init
This commit is contained in:
Vakula Uladimir
@@ -0,0 +1,207 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { ScraperService } from '#services/scraper_service'
|
||||
import { NotificationService } from '#services/notification_service'
|
||||
import Auction from '#models/auction'
|
||||
import ParseLog from '#models/parse_log'
|
||||
import { DateTime } from 'luxon'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/**
|
||||
* Command to parse auctions from icetrade.by
|
||||
*
|
||||
* Usage:
|
||||
* node ace parse:auctions
|
||||
* node ace parse:auctions --pages=5
|
||||
* node ace parse:auctions --pages=3 --skip-notifications
|
||||
*/
|
||||
export default class ParseAuctions extends BaseCommand {
|
||||
static commandName = 'parse:auctions'
|
||||
static description = 'Scrape auctions from icetrade.by and store them in the database'
|
||||
static options: CommandOptions = {
|
||||
startApp: true,
|
||||
allowUnknownFlags: false,
|
||||
}
|
||||
|
||||
@flags.number({
|
||||
description: 'Number of pages to scrape',
|
||||
default: 1,
|
||||
alias: 'p',
|
||||
})
|
||||
declare pages: number
|
||||
|
||||
@flags.boolean({
|
||||
description: 'Skip sending notifications after parsing',
|
||||
default: false,
|
||||
})
|
||||
declare skipNotifications: boolean
|
||||
|
||||
async run() {
|
||||
const startTime = DateTime.now()
|
||||
const scraper = new ScraperService()
|
||||
const notificationService = new NotificationService()
|
||||
|
||||
// Create ParseLog entry with status 'running'
|
||||
const parseLog = await ParseLog.create({
|
||||
parseType: 'auction',
|
||||
status: 'running',
|
||||
itemsFound: 0,
|
||||
errors: null,
|
||||
startedAt: startTime,
|
||||
completedAt: null,
|
||||
})
|
||||
|
||||
this.logger.info(`Starting auction parsing (log ID: ${parseLog.id})`)
|
||||
this.logger.info(`Scraping ${this.pages} page(s) from icetrade.by`)
|
||||
|
||||
let totalScraped = 0
|
||||
let newAuctions = 0
|
||||
let updatedAuctions = 0
|
||||
let errors: string[] = []
|
||||
|
||||
try {
|
||||
// Step 1: Scrape auctions
|
||||
this.logger.info('Step 1: Scraping auctions...')
|
||||
const auctionData = await scraper.scrapeAuctions(this.pages)
|
||||
totalScraped = auctionData.length
|
||||
|
||||
this.logger.success(`Scraped ${totalScraped} auction(s)`)
|
||||
|
||||
if (totalScraped === 0) {
|
||||
this.logger.warning('No auctions found to process')
|
||||
await this.updateParseLog(parseLog, 'completed', 0, null)
|
||||
return
|
||||
}
|
||||
|
||||
// Step 2: Upsert auctions to database
|
||||
this.logger.info('Step 2: Saving auctions to database...')
|
||||
const processedAuctions: Auction[] = []
|
||||
|
||||
for (const data of auctionData) {
|
||||
try {
|
||||
// Check if auction exists by auctionNum
|
||||
const existingAuction = await Auction.query()
|
||||
.where('auctionNum', data.auctionNum)
|
||||
.first()
|
||||
|
||||
let auction: Auction
|
||||
|
||||
if (existingAuction) {
|
||||
// Update existing auction
|
||||
existingAuction.title = data.title
|
||||
existingAuction.description = data.description
|
||||
existingAuction.organization = data.organization
|
||||
existingAuction.status = data.status
|
||||
existingAuction.deadline = data.deadline ? DateTime.fromISO(data.deadline) : null
|
||||
existingAuction.url = data.link
|
||||
existingAuction.rawData = data
|
||||
|
||||
await existingAuction.save()
|
||||
auction = existingAuction
|
||||
updatedAuctions++
|
||||
|
||||
logger.debug(`Updated auction: ${data.auctionNum}`)
|
||||
} else {
|
||||
// Create new auction
|
||||
auction = await Auction.create({
|
||||
auctionNum: data.auctionNum,
|
||||
title: data.title,
|
||||
description: data.description,
|
||||
organization: data.organization,
|
||||
status: data.status,
|
||||
price: null, // Price not available in current schema
|
||||
deadline: data.deadline ? DateTime.fromISO(data.deadline) : null,
|
||||
url: data.link,
|
||||
rawData: data,
|
||||
})
|
||||
|
||||
newAuctions++
|
||||
logger.debug(`Created auction: ${data.auctionNum}`)
|
||||
|
||||
// Step 3: Check for keyword matches and send notifications (for new auctions only)
|
||||
if (!this.skipNotifications) {
|
||||
try {
|
||||
await notificationService.checkAndNotify(auction)
|
||||
} catch (error) {
|
||||
const errorMsg = `Notification check failed for auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
|
||||
logger.error(errorMsg)
|
||||
errors.push(errorMsg)
|
||||
// Don't fail the entire process if notification fails
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
processedAuctions.push(auction)
|
||||
} catch (error) {
|
||||
const errorMsg = `Failed to save auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
|
||||
logger.error(errorMsg)
|
||||
errors.push(errorMsg)
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.success(
|
||||
`Saved ${processedAuctions.length} auction(s): ${newAuctions} new, ${updatedAuctions} updated`
|
||||
)
|
||||
|
||||
if (this.skipNotifications) {
|
||||
this.logger.info('Notifications skipped (--skip-notifications flag set)')
|
||||
}
|
||||
|
||||
// Update ParseLog with success status
|
||||
const status = errors.length > 0 ? 'completed_with_errors' : 'completed'
|
||||
await this.updateParseLog(
|
||||
parseLog,
|
||||
status,
|
||||
processedAuctions.length,
|
||||
errors.length > 0 ? errors.join('\n') : null
|
||||
)
|
||||
|
||||
// Summary
|
||||
const duration = DateTime.now().diff(startTime).toFormat("m 'min' s 'sec'")
|
||||
this.logger.info('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━')
|
||||
this.logger.info('Parsing Summary:')
|
||||
this.logger.info(` Duration: ${duration}`)
|
||||
this.logger.info(` Total scraped: ${totalScraped}`)
|
||||
this.logger.info(` New auctions: ${newAuctions}`)
|
||||
this.logger.info(` Updated auctions: ${updatedAuctions}`)
|
||||
if (errors.length > 0) {
|
||||
this.logger.warning(` Errors: ${errors.length}`)
|
||||
}
|
||||
this.logger.info('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━')
|
||||
|
||||
if (errors.length > 0) {
|
||||
this.logger.warning('Parsing completed with errors')
|
||||
this.exitCode = 1
|
||||
} else {
|
||||
this.logger.success('Parsing completed successfully')
|
||||
}
|
||||
} catch (error) {
|
||||
// Handle catastrophic failure
|
||||
const errorMsg = error instanceof Error ? error.message : String(error)
|
||||
logger.error('Fatal error during auction parsing', { error: errorMsg })
|
||||
|
||||
await this.updateParseLog(parseLog, 'failed', 0, errorMsg)
|
||||
|
||||
this.logger.error('Parsing failed: ' + errorMsg)
|
||||
this.exitCode = 1
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update ParseLog entry with final status
|
||||
*/
|
||||
private async updateParseLog(
|
||||
parseLog: ParseLog,
|
||||
status: string,
|
||||
itemsFound: number,
|
||||
errors: string | null
|
||||
): Promise<void> {
|
||||
parseLog.status = status
|
||||
parseLog.itemsFound = itemsFound
|
||||
parseLog.errors = errors
|
||||
parseLog.completedAt = DateTime.now()
|
||||
await parseLog.save()
|
||||
|
||||
logger.info(`Updated ParseLog ${parseLog.id}: status=${status}, items=${itemsFound}`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { TelegramService } from '#services/telegram_service'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/**
|
||||
* Command to start Telegram bot
|
||||
*
|
||||
* Usage:
|
||||
* node ace telegram:start
|
||||
*
|
||||
* This command runs as a long-lived process and handles graceful shutdown.
|
||||
*/
|
||||
export default class StartTelegramBot extends BaseCommand {
|
||||
static commandName = 'telegram:start'
|
||||
static description = 'Start the Telegram bot for auction notifications'
|
||||
|
||||
static options: CommandOptions = {
|
||||
startApp: true,
|
||||
staysAlive: true,
|
||||
}
|
||||
|
||||
private telegramService: TelegramService | null = null
|
||||
|
||||
async run() {
|
||||
try {
|
||||
this.logger.info('Initializing Telegram bot service...')
|
||||
|
||||
// Get singleton instance
|
||||
this.telegramService = TelegramService.getInstance()
|
||||
|
||||
// Setup graceful shutdown handlers
|
||||
this.setupShutdownHandlers()
|
||||
|
||||
// Start the bot
|
||||
this.logger.info('Starting Telegram bot...')
|
||||
await this.telegramService.start()
|
||||
|
||||
this.logger.success('Telegram bot is running! Press Ctrl+C to stop.')
|
||||
} catch (error) {
|
||||
const errorMsg = error instanceof Error ? error.message : String(error)
|
||||
this.logger.error(`Failed to start Telegram bot: ${errorMsg}`)
|
||||
logger.error('Telegram bot startup failed', { error: errorMsg })
|
||||
this.exitCode = 1
|
||||
await this.terminate()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup handlers for graceful shutdown
|
||||
*/
|
||||
private setupShutdownHandlers(): void {
|
||||
const shutdown = async (signal: string) => {
|
||||
this.logger.info(`Received ${signal}, shutting down gracefully...`)
|
||||
|
||||
if (this.telegramService) {
|
||||
try {
|
||||
await this.telegramService.stop()
|
||||
this.logger.success('Telegram bot stopped successfully')
|
||||
} catch (error) {
|
||||
const errorMsg = error instanceof Error ? error.message : String(error)
|
||||
this.logger.error(`Error during shutdown: ${errorMsg}`)
|
||||
logger.error('Telegram bot shutdown error', { error: errorMsg })
|
||||
}
|
||||
}
|
||||
|
||||
await this.terminate()
|
||||
}
|
||||
|
||||
// Handle SIGINT (Ctrl+C)
|
||||
process.on('SIGINT', () => shutdown('SIGINT'))
|
||||
|
||||
// Handle SIGTERM (kill command)
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'))
|
||||
}
|
||||
|
||||
/**
|
||||
* Command termination handler
|
||||
*/
|
||||
async completed() {
|
||||
if (this.telegramService && this.telegramService.getIsRunning()) {
|
||||
this.logger.info('Stopping Telegram bot...')
|
||||
await this.telegramService.stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/**
|
||||
* Test network connectivity to icetrade.by
|
||||
*
|
||||
* Usage:
|
||||
* node ace test:connection
|
||||
*/
|
||||
export default class TestConnection extends BaseCommand {
|
||||
static commandName = 'test:connection'
|
||||
static description = 'Test network connectivity to icetrade.by'
|
||||
static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
async run() {
|
||||
const testUrl = 'https://icetrade.by'
|
||||
|
||||
this.logger.info('Testing network connectivity...')
|
||||
this.logger.info(`Target URL: ${testUrl}`)
|
||||
|
||||
try {
|
||||
// Test 1: Basic DNS resolution
|
||||
this.logger.info('Test 1: DNS resolution')
|
||||
try {
|
||||
const dns = await import('node:dns/promises')
|
||||
const addresses = await dns.resolve4('icetrade.by')
|
||||
this.logger.success(`DNS resolved: ${addresses.join(', ')}`)
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`DNS resolution failed: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
throw error
|
||||
}
|
||||
|
||||
// Test 2: Basic HTTP connection
|
||||
this.logger.info('Test 2: HTTP connection')
|
||||
const startTime = Date.now()
|
||||
|
||||
const controller = new AbortController()
|
||||
const timeoutId = setTimeout(() => controller.abort(), 30000)
|
||||
|
||||
try {
|
||||
const response = await fetch(testUrl, {
|
||||
method: 'HEAD',
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
},
|
||||
signal: controller.signal,
|
||||
})
|
||||
|
||||
clearTimeout(timeoutId)
|
||||
const duration = Date.now() - startTime
|
||||
|
||||
this.logger.success(`HTTP response: ${response.status} ${response.statusText}`)
|
||||
this.logger.info(`Response time: ${duration}ms`)
|
||||
this.logger.info(`Content-Type: ${response.headers.get('content-type')}`)
|
||||
this.logger.info(`Server: ${response.headers.get('server')}`)
|
||||
} catch (error) {
|
||||
clearTimeout(timeoutId)
|
||||
const err = error instanceof Error ? error : new Error(String(error))
|
||||
|
||||
this.logger.error(`HTTP connection failed: ${err.message}`)
|
||||
|
||||
// Log detailed error info
|
||||
logger.error('Connection test failed', {
|
||||
name: err.name,
|
||||
message: err.message,
|
||||
stack: err.stack,
|
||||
cause: err.cause,
|
||||
})
|
||||
|
||||
throw error
|
||||
}
|
||||
|
||||
// Test 3: Full page fetch
|
||||
this.logger.info('Test 3: Full page fetch')
|
||||
try {
|
||||
const fullResponse = await fetch(testUrl, {
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
Accept: 'text/html',
|
||||
},
|
||||
})
|
||||
|
||||
const text = await fullResponse.text()
|
||||
this.logger.success(`Received ${text.length} bytes`)
|
||||
this.logger.info(`First 200 chars: ${text.substring(0, 200).replace(/\s+/g, ' ')}`)
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Full fetch failed: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
throw error
|
||||
}
|
||||
|
||||
// Test 4: Actual auction page
|
||||
this.logger.info('Test 4: Auction list page')
|
||||
const auctionUrl = 'https://icetrade.by/trades/index?onPage=100&p=1'
|
||||
try {
|
||||
const auctionResponse = await fetch(auctionUrl, {
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
Accept: 'text/html',
|
||||
'Accept-Language': 'ru-RU,ru;q=0.9',
|
||||
},
|
||||
})
|
||||
|
||||
if (!auctionResponse.ok) {
|
||||
throw new Error(
|
||||
`HTTP ${auctionResponse.status}: ${auctionResponse.statusText}`
|
||||
)
|
||||
}
|
||||
|
||||
const html = await auctionResponse.text()
|
||||
this.logger.success(`Auction page received: ${html.length} bytes`)
|
||||
|
||||
// Check for auction table
|
||||
if (html.includes('table.auctions') || html.includes('class="auctions')) {
|
||||
this.logger.success('Auction table found in HTML')
|
||||
} else {
|
||||
this.logger.warning('Auction table not found in HTML (page structure may have changed)')
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Auction page fetch failed: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
throw error
|
||||
}
|
||||
|
||||
this.logger.success('All connectivity tests passed!')
|
||||
} catch (error) {
|
||||
this.logger.error('Connectivity test failed')
|
||||
this.exitCode = 1
|
||||
|
||||
// Print environment info
|
||||
this.logger.info('Environment info:')
|
||||
this.logger.info(` NODE_ENV: ${process.env.NODE_ENV}`)
|
||||
this.logger.info(` Platform: ${process.platform}`)
|
||||
this.logger.info(` Node version: ${process.version}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user