auction-scrapper/commands/parse_auctions.ts

import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import { ScraperService } from '#services/scraper_service'
import { NotificationService } from '#services/notification_service'
import Auction from '#models/auction'
import ParseLog from '#models/parse_log'
import { DateTime } from 'luxon'
import logger from '@adonisjs/core/services/logger'

/**
 * Command to parse auctions from icetrade.by
 *
 * Usage:
 *   node ace parse:auctions
 *   node ace parse:auctions --pages=5
 *   node ace parse:auctions --pages=3 --skip-notifications
 */
export default class ParseAuctions extends BaseCommand {
  static commandName = 'parse:auctions'
  static description = 'Scrape auctions from icetrade.by and store them in the database'
  static options: CommandOptions = {
    startApp: true,
    allowUnknownFlags: false,
  }

  @flags.number({
    description: 'Number of pages to scrape',
    default: 1,
    alias: 'p',
  })
  declare pages: number

  @flags.boolean({
    description: 'Skip sending notifications after parsing',
    default: false,
  })
  declare skipNotifications: boolean

  async run() {
    const startTime = DateTime.now()
    const scraper = new ScraperService()
    const notificationService = new NotificationService()

    // Create ParseLog entry with status 'running'
    const parseLog = await ParseLog.create({
      parseType: 'auction',
      status: 'running',
      itemsFound: 0,
      errors: null,
      startedAt: startTime,
      completedAt: null,
    })

    this.logger.info(`Starting auction parsing (log ID: ${parseLog.id})`)
    this.logger.info(`Scraping ${this.pages} page(s) from icetrade.by`)

    let totalScraped = 0
    let newAuctions = 0
    let updatedAuctions = 0
    let errors: string[] = []

    try {
      // Step 1: Scrape auctions
      this.logger.info('Step 1: Scraping auctions...')
      const auctionData = await scraper.scrapeAuctions(this.pages)
      totalScraped = auctionData.length

      this.logger.success(`Scraped ${totalScraped} auction(s)`)

      if (totalScraped === 0) {
        this.logger.warning('No auctions found to process')
        await this.updateParseLog(parseLog, 'completed', 0, null)
        return
      }

      // Step 2: Upsert auctions to database
      this.logger.info('Step 2: Saving auctions to database...')
      const processedAuctions: Auction[] = []

      for (const data of auctionData) {
        try {
          // Check if auction exists by auctionNum
          const existingAuction = await Auction.query()
            .where('auctionNum', data.auctionNum)
            .first()

          let auction: Auction

          if (existingAuction) {
            // Update existing auction
            existingAuction.title = data.title
            existingAuction.description = data.description
            existingAuction.organization = data.organization
            existingAuction.status = data.status
            existingAuction.deadline = data.deadline ? DateTime.fromISO(data.deadline) : null
            existingAuction.url = data.link
            existingAuction.rawData = data

            await existingAuction.save()
            auction = existingAuction
            updatedAuctions++

            logger.debug(`Updated auction: ${data.auctionNum}`)
          } else {
            // Create new auction
            auction = await Auction.create({
              auctionNum: data.auctionNum,
              title: data.title,
              description: data.description,
              organization: data.organization,
              status: data.status,
              price: null, // Price not available in current schema
              deadline: data.deadline ? DateTime.fromISO(data.deadline) : null,
              url: data.link,
              rawData: data,
            })

            newAuctions++
            logger.debug(`Created auction: ${data.auctionNum}`)

            // Step 3: Check for keyword matches and send notifications (for new auctions only)
            if (!this.skipNotifications) {
              try {
                await notificationService.checkAndNotify(auction)
              } catch (error) {
                const errorMsg = `Notification check failed for auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
                logger.error(errorMsg)
                errors.push(errorMsg)
                // Don't fail the entire process if notification fails
              }
            }
          }

          processedAuctions.push(auction)
        } catch (error) {
          const errorMsg = `Failed to save auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
          logger.error(errorMsg)
          errors.push(errorMsg)
        }
      }

      this.logger.success(
        `Saved ${processedAuctions.length} auction(s): ${newAuctions} new, ${updatedAuctions} updated`
      )

      if (this.skipNotifications) {
        this.logger.info('Notifications skipped (--skip-notifications flag set)')
      }

      // Update ParseLog with success status
      const status = errors.length > 0 ? 'completed_with_errors' : 'completed'
      await this.updateParseLog(
        parseLog,
        status,
        processedAuctions.length,
        errors.length > 0 ? errors.join('\n') : null
      )

      // Summary
      const duration = DateTime.now().diff(startTime).toFormat("m 'min' s 'sec'")
      this.logger.info('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━')
      this.logger.info('Parsing Summary:')
      this.logger.info(`  Duration: ${duration}`)
      this.logger.info(`  Total scraped: ${totalScraped}`)
      this.logger.info(`  New auctions: ${newAuctions}`)
      this.logger.info(`  Updated auctions: ${updatedAuctions}`)
      if (errors.length > 0) {
        this.logger.warning(`  Errors: ${errors.length}`)
      }
      this.logger.info('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━')

      if (errors.length > 0) {
        this.logger.warning('Parsing completed with errors')
        this.exitCode = 1
      } else {
        this.logger.success('Parsing completed successfully')
      }
    } catch (error) {
      // Handle catastrophic failure
      const errorMsg = error instanceof Error ? error.message : String(error)
      logger.error('Fatal error during auction parsing', { error: errorMsg })

      await this.updateParseLog(parseLog, 'failed', 0, errorMsg)

      this.logger.error('Parsing failed: ' + errorMsg)
      this.exitCode = 1
    }
  }

  /**
   * Update ParseLog entry with final status
   */
  private async updateParseLog(
    parseLog: ParseLog,
    status: string,
    itemsFound: number,
    errors: string | null
  ): Promise<void> {
    parseLog.status = status
    parseLog.itemsFound = itemsFound
    parseLog.errors = errors
    parseLog.completedAt = DateTime.now()
    await parseLog.save()

    logger.info(`Updated ParseLog ${parseLog.id}: status=${status}, items=${itemsFound}`)
  }
}