import { BaseCommand, flags } from '@adonisjs/core/ace' import type { CommandOptions } from '@adonisjs/core/types/ace' import { ScraperService } from '#services/scraper_service' import { NotificationService } from '#services/notification_service' import Auction from '#models/auction' import ParseLog from '#models/parse_log' import { DateTime } from 'luxon' import logger from '@adonisjs/core/services/logger' /** * Command to parse auctions from icetrade.by * * Usage: * node ace parse:auctions * node ace parse:auctions --pages=5 * node ace parse:auctions --pages=3 --skip-notifications */ export default class ParseAuctions extends BaseCommand { static commandName = 'parse:auctions' static description = 'Scrape auctions from icetrade.by and store them in the database' static options: CommandOptions = { startApp: true, allowUnknownFlags: false, } @flags.number({ description: 'Number of pages to scrape', default: 1, alias: 'p', }) declare pages: number @flags.boolean({ description: 'Skip sending notifications after parsing', default: false, }) declare skipNotifications: boolean async run() { const startTime = DateTime.now() const scraper = new ScraperService() const notificationService = new NotificationService() // Create ParseLog entry with status 'running' const parseLog = await ParseLog.create({ parseType: 'auction', status: 'running', itemsFound: 0, errors: null, startedAt: startTime, completedAt: null, }) this.logger.info(`Starting auction parsing (log ID: ${parseLog.id})`) this.logger.info(`Scraping ${this.pages} page(s) from icetrade.by`) let totalScraped = 0 let newAuctions = 0 let updatedAuctions = 0 let errors: string[] = [] try { // Step 1: Scrape auctions this.logger.info('Step 1: Scraping auctions...') const auctionData = await scraper.scrapeAuctions(this.pages) totalScraped = auctionData.length this.logger.success(`Scraped ${totalScraped} auction(s)`) if (totalScraped === 0) { this.logger.warning('No auctions found to process') await this.updateParseLog(parseLog, 'completed', 0, null) return } // Step 2: Upsert auctions to database this.logger.info('Step 2: Saving auctions to database...') const processedAuctions: Auction[] = [] for (const data of auctionData) { try { // Check if auction exists by auctionNum const existingAuction = await Auction.query() .where('auctionNum', data.auctionNum) .first() let auction: Auction if (existingAuction) { // Update existing auction existingAuction.title = data.title existingAuction.description = data.description existingAuction.organization = data.organization existingAuction.status = data.status existingAuction.deadline = data.deadline ? DateTime.fromISO(data.deadline) : null existingAuction.url = data.link existingAuction.rawData = data await existingAuction.save() auction = existingAuction updatedAuctions++ logger.debug(`Updated auction: ${data.auctionNum}`) } else { // Create new auction auction = await Auction.create({ auctionNum: data.auctionNum, title: data.title, description: data.description, organization: data.organization, status: data.status, price: null, // Price not available in current schema deadline: data.deadline ? DateTime.fromISO(data.deadline) : null, url: data.link, rawData: data, }) newAuctions++ logger.debug(`Created auction: ${data.auctionNum}`) // Step 3: Check for keyword matches and send notifications (for new auctions only) if (!this.skipNotifications) { try { await notificationService.checkAndNotify(auction) } catch (error) { const errorMsg = `Notification check failed for auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}` logger.error(errorMsg) errors.push(errorMsg) // Don't fail the entire process if notification fails } } } processedAuctions.push(auction) } catch (error) { const errorMsg = `Failed to save auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}` logger.error(errorMsg) errors.push(errorMsg) } } this.logger.success( `Saved ${processedAuctions.length} auction(s): ${newAuctions} new, ${updatedAuctions} updated` ) if (this.skipNotifications) { this.logger.info('Notifications skipped (--skip-notifications flag set)') } // Update ParseLog with success status const status = errors.length > 0 ? 'completed_with_errors' : 'completed' await this.updateParseLog( parseLog, status, processedAuctions.length, errors.length > 0 ? errors.join('\n') : null ) // Summary const duration = DateTime.now().diff(startTime).toFormat("m 'min' s 'sec'") this.logger.info('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━') this.logger.info('Parsing Summary:') this.logger.info(` Duration: ${duration}`) this.logger.info(` Total scraped: ${totalScraped}`) this.logger.info(` New auctions: ${newAuctions}`) this.logger.info(` Updated auctions: ${updatedAuctions}`) if (errors.length > 0) { this.logger.warning(` Errors: ${errors.length}`) } this.logger.info('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━') if (errors.length > 0) { this.logger.warning('Parsing completed with errors') this.exitCode = 1 } else { this.logger.success('Parsing completed successfully') } } catch (error) { // Handle catastrophic failure const errorMsg = error instanceof Error ? error.message : String(error) logger.error('Fatal error during auction parsing', { error: errorMsg }) await this.updateParseLog(parseLog, 'failed', 0, errorMsg) this.logger.error('Parsing failed: ' + errorMsg) this.exitCode = 1 } } /** * Update ParseLog entry with final status */ private async updateParseLog( parseLog: ParseLog, status: string, itemsFound: number, errors: string | null ): Promise { parseLog.status = status parseLog.itemsFound = itemsFound parseLog.errors = errors parseLog.completedAt = DateTime.now() await parseLog.save() logger.info(`Updated ParseLog ${parseLog.id}: status=${status}, items=${itemsFound}`) } }