init
This commit is contained in:
@@ -0,0 +1,324 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import Auction from '#models/auction'
|
||||
import ParseLog from '#models/parse_log'
|
||||
import { errors as lucidErrors } from '@adonisjs/lucid'
|
||||
import { DateTime } from 'luxon'
|
||||
import { ScraperService } from '#services/scraper_service'
|
||||
import { NotificationService } from '#services/notification_service'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
export default class AuctionsController {
|
||||
/**
|
||||
* List auctions with pagination
|
||||
* GET /api/auctions?page=1&limit=20
|
||||
*/
|
||||
async index({ request, response }: HttpContext) {
|
||||
const page = request.input('page', 1)
|
||||
const limit = request.input('limit', 20)
|
||||
|
||||
const pageNumber = Math.max(1, Number(page))
|
||||
const limitNumber = Math.min(100, Math.max(1, Number(limit)))
|
||||
|
||||
try {
|
||||
const auctions = await Auction.query()
|
||||
.orderBy('created_at', 'desc')
|
||||
.paginate(pageNumber, limitNumber)
|
||||
|
||||
return response.ok(auctions.serialize())
|
||||
} catch (error) {
|
||||
return response.internalServerError({
|
||||
error: 'Failed to fetch auctions',
|
||||
message: error instanceof Error ? error.message : 'Unknown error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single auction by id with relationships
|
||||
* GET /api/auctions/:id
|
||||
*/
|
||||
async show({ params, response }: HttpContext) {
|
||||
const auctionId = params.id
|
||||
|
||||
if (!auctionId || isNaN(Number(auctionId))) {
|
||||
return response.badRequest({
|
||||
error: 'Invalid auction id',
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
const auction = await Auction.query()
|
||||
.where('id', auctionId)
|
||||
.preload('notifications')
|
||||
.firstOrFail()
|
||||
|
||||
return response.ok(auction.serialize())
|
||||
} catch (error) {
|
||||
if (error instanceof lucidErrors.E_ROW_NOT_FOUND) {
|
||||
return response.notFound({
|
||||
error: 'Auction not found',
|
||||
})
|
||||
}
|
||||
|
||||
return response.internalServerError({
|
||||
error: 'Failed to fetch auction',
|
||||
message: error instanceof Error ? error.message : 'Unknown error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search auctions by keyword/title/status
|
||||
* GET /api/auctions/search?q=keyword&status=active&page=1&limit=20
|
||||
*/
|
||||
async search({ request, response }: HttpContext) {
|
||||
const searchQuery = request.input('q', '')
|
||||
const status = request.input('status')
|
||||
const page = request.input('page', 1)
|
||||
const limit = request.input('limit', 20)
|
||||
|
||||
const pageNumber = Math.max(1, Number(page))
|
||||
const limitNumber = Math.min(100, Math.max(1, Number(limit)))
|
||||
|
||||
try {
|
||||
const query = Auction.query()
|
||||
|
||||
if (searchQuery && typeof searchQuery === 'string' && searchQuery.trim().length > 0) {
|
||||
const trimmedQuery = searchQuery.trim()
|
||||
query.where((builder) => {
|
||||
builder
|
||||
.whereILike('title', `%${trimmedQuery}%`)
|
||||
.orWhereILike('description', `%${trimmedQuery}%`)
|
||||
.orWhereILike('auction_num', `%${trimmedQuery}%`)
|
||||
.orWhereILike('organization', `%${trimmedQuery}%`)
|
||||
})
|
||||
}
|
||||
|
||||
if (status && typeof status === 'string' && status.trim().length > 0) {
|
||||
query.where('status', status.trim())
|
||||
}
|
||||
|
||||
const auctions = await query.orderBy('created_at', 'desc').paginate(pageNumber, limitNumber)
|
||||
|
||||
return response.ok(auctions.serialize())
|
||||
} catch (error) {
|
||||
return response.internalServerError({
|
||||
error: 'Failed to search auctions',
|
||||
message: error instanceof Error ? error.message : 'Unknown error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List all auctions from last 3 days
|
||||
* GET /list
|
||||
*/
|
||||
async list({ response }: HttpContext) {
|
||||
try {
|
||||
const threeDaysAgo = DateTime.now().minus({ days: 3 })
|
||||
|
||||
const auctions = await Auction.query()
|
||||
.where('created_at', '>=', threeDaysAgo.toSQL())
|
||||
.orderBy('created_at', 'desc')
|
||||
|
||||
return response.ok({
|
||||
data: auctions.map((auction) => auction.serialize()),
|
||||
meta: {
|
||||
total: auctions.length,
|
||||
from_date: threeDaysAgo.toISO(),
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return response.internalServerError({
|
||||
error: 'Failed to fetch recent auctions',
|
||||
message: error instanceof Error ? error.message : 'Unknown error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Render HTML view of auctions from last 3 days
|
||||
* GET /list-view
|
||||
*/
|
||||
async listView({ view }: HttpContext) {
|
||||
const threeDaysAgo = DateTime.now().minus({ days: 3 })
|
||||
|
||||
const auctions = await Auction.query()
|
||||
.where('created_at', '>=', threeDaysAgo.toSQL())
|
||||
.orderBy('created_at', 'desc')
|
||||
|
||||
return view.render('auctions/list', {
|
||||
auctions,
|
||||
fromDate: threeDaysAgo.toFormat('dd.MM.yyyy HH:mm'),
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger auction parsing from web interface
|
||||
* POST /trigger-parse
|
||||
*/
|
||||
async triggerParse({ request, response, session }: HttpContext) {
|
||||
try {
|
||||
// Validate input
|
||||
const pages = request.input('pages', 1)
|
||||
const notifySubscribers = request.input('notifySubscribers') === 'on'
|
||||
|
||||
const pagesNumber = Math.max(1, Math.min(10, Number(pages)))
|
||||
|
||||
if (isNaN(pagesNumber)) {
|
||||
session.flash('error', 'Invalid number of pages')
|
||||
return response.redirect('/list-view')
|
||||
}
|
||||
|
||||
logger.info(`Parsing triggered from web interface`, {
|
||||
pages: pagesNumber,
|
||||
notifySubscribers,
|
||||
})
|
||||
|
||||
const startTime = DateTime.now()
|
||||
const scraper = new ScraperService()
|
||||
const notificationService = new NotificationService()
|
||||
|
||||
// Create ParseLog entry
|
||||
const parseLog = await ParseLog.create({
|
||||
parseType: 'auction',
|
||||
status: 'running',
|
||||
itemsFound: 0,
|
||||
errors: null,
|
||||
startedAt: startTime,
|
||||
completedAt: null,
|
||||
})
|
||||
|
||||
let newAuctions = 0
|
||||
let updatedAuctions = 0
|
||||
let totalScraped = 0
|
||||
const errors: string[] = []
|
||||
|
||||
// Step 1: Scrape auctions
|
||||
logger.info(`Scraping ${pagesNumber} page(s) from icetrade.by`)
|
||||
const auctionData = await scraper.scrapeAuctions(pagesNumber)
|
||||
totalScraped = auctionData.length
|
||||
|
||||
if (totalScraped === 0) {
|
||||
await this.updateParseLog(parseLog, 'completed', 0, null)
|
||||
session.flash('error', 'No auctions found during parsing')
|
||||
return response.redirect('/list-view')
|
||||
}
|
||||
|
||||
// Step 2: Upsert auctions to database
|
||||
logger.info(`Saving ${totalScraped} auction(s) to database`)
|
||||
|
||||
for (const data of auctionData) {
|
||||
try {
|
||||
// Check if auction exists by auctionNum
|
||||
const existingAuction = await Auction.query()
|
||||
.where('auctionNum', data.auctionNum)
|
||||
.first()
|
||||
|
||||
let auction: Auction
|
||||
|
||||
if (existingAuction) {
|
||||
// Update existing auction
|
||||
existingAuction.title = data.title
|
||||
existingAuction.description = data.description
|
||||
existingAuction.organization = data.organization
|
||||
existingAuction.status = data.status
|
||||
existingAuction.deadline = data.deadline ? DateTime.fromISO(data.deadline) : null
|
||||
existingAuction.url = data.link
|
||||
existingAuction.rawData = data
|
||||
|
||||
await existingAuction.save()
|
||||
auction = existingAuction
|
||||
updatedAuctions++
|
||||
|
||||
logger.debug(`Updated auction: ${data.auctionNum}`)
|
||||
} else {
|
||||
// Create new auction
|
||||
auction = await Auction.create({
|
||||
auctionNum: data.auctionNum,
|
||||
title: data.title,
|
||||
description: data.description,
|
||||
organization: data.organization,
|
||||
status: data.status,
|
||||
price: null,
|
||||
deadline: data.deadline ? DateTime.fromISO(data.deadline) : null,
|
||||
url: data.link,
|
||||
rawData: data,
|
||||
})
|
||||
|
||||
newAuctions++
|
||||
logger.debug(`Created auction: ${data.auctionNum}`)
|
||||
|
||||
// Step 3: Check for keyword matches and send notifications (for new auctions only)
|
||||
if (notifySubscribers) {
|
||||
try {
|
||||
await notificationService.checkAndNotify(auction)
|
||||
} catch (error) {
|
||||
const errorMsg = `Notification check failed for auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
|
||||
logger.error(errorMsg)
|
||||
errors.push(errorMsg)
|
||||
// Don't fail the entire process if notification fails
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMsg = `Failed to save auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
|
||||
logger.error(errorMsg)
|
||||
errors.push(errorMsg)
|
||||
}
|
||||
}
|
||||
|
||||
// Update ParseLog with success status
|
||||
const status = errors.length > 0 ? 'completed_with_errors' : 'completed'
|
||||
await this.updateParseLog(
|
||||
parseLog,
|
||||
status,
|
||||
newAuctions + updatedAuctions,
|
||||
errors.length > 0 ? errors.join('\n') : null
|
||||
)
|
||||
|
||||
// Create success message
|
||||
let successMessage = `Parsing completed: ${totalScraped} auctions scraped, ${newAuctions} new, ${updatedAuctions} updated`
|
||||
|
||||
if (!notifySubscribers) {
|
||||
successMessage += ' (notifications disabled)'
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
successMessage += ` with ${errors.length} error(s)`
|
||||
}
|
||||
|
||||
session.flash('success', successMessage)
|
||||
logger.info(successMessage)
|
||||
|
||||
return response.redirect('/list-view')
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error during parsing'
|
||||
|
||||
logger.error('Error in triggerParse', {
|
||||
error: errorMessage,
|
||||
})
|
||||
|
||||
session.flash('error', `Parsing failed: ${errorMessage}`)
|
||||
return response.redirect('/list-view')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update ParseLog entry with final status
|
||||
*/
|
||||
private async updateParseLog(
|
||||
parseLog: ParseLog,
|
||||
status: string,
|
||||
itemsFound: number,
|
||||
errors: string | null
|
||||
): Promise<void> {
|
||||
parseLog.status = status
|
||||
parseLog.itemsFound = itemsFound
|
||||
parseLog.errors = errors
|
||||
parseLog.completedAt = DateTime.now()
|
||||
await parseLog.save()
|
||||
|
||||
logger.info(`Updated ParseLog ${parseLog.id}: status=${status}, items=${itemsFound}`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import Keyword from '#models/keyword'
|
||||
|
||||
export default class KeywordsController {
|
||||
/**
|
||||
* List keywords by user_id with optional filtering by is_active
|
||||
* GET /api/keywords?user_id=1&is_active=true
|
||||
*/
|
||||
async index({ request, response }: HttpContext) {
|
||||
const userId = request.input('user_id')
|
||||
const isActive = request.input('is_active')
|
||||
|
||||
if (!userId) {
|
||||
return response.badRequest({
|
||||
error: 'user_id query parameter is required',
|
||||
})
|
||||
}
|
||||
|
||||
const query = Keyword.query().where('user_id', userId)
|
||||
|
||||
if (isActive !== undefined) {
|
||||
const activeValue = isActive === 'true' || isActive === true
|
||||
query.where('is_active', activeValue)
|
||||
}
|
||||
|
||||
const keywords = await query.orderBy('created_at', 'desc')
|
||||
|
||||
return response.ok(keywords)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new keyword
|
||||
* POST /api/keywords
|
||||
* Body: { user_id: number, keyword: string, case_sensitive?: boolean }
|
||||
*/
|
||||
async store({ request, response }: HttpContext) {
|
||||
const userId = request.input('user_id')
|
||||
const keyword = request.input('keyword')
|
||||
const caseSensitive = request.input('case_sensitive', false)
|
||||
|
||||
if (!userId || !keyword) {
|
||||
return response.badRequest({
|
||||
error: 'user_id and keyword are required',
|
||||
})
|
||||
}
|
||||
|
||||
if (typeof keyword !== 'string' || keyword.trim().length === 0) {
|
||||
return response.badRequest({
|
||||
error: 'keyword must be a non-empty string',
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
const newKeyword = await Keyword.create({
|
||||
userId,
|
||||
keyword: keyword.trim(),
|
||||
caseSensitive,
|
||||
isActive: true,
|
||||
})
|
||||
|
||||
return response.created(newKeyword)
|
||||
} catch (error) {
|
||||
return response.internalServerError({
|
||||
error: 'Failed to create keyword',
|
||||
message: error instanceof Error ? error.message : 'Unknown error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a keyword by id
|
||||
* DELETE /api/keywords/:id
|
||||
*/
|
||||
async destroy({ params, response }: HttpContext) {
|
||||
const keywordId = params.id
|
||||
|
||||
if (!keywordId || isNaN(Number(keywordId))) {
|
||||
return response.badRequest({
|
||||
error: 'Invalid keyword id',
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
const keyword = await Keyword.findOrFail(keywordId)
|
||||
await keyword.delete()
|
||||
|
||||
return response.noContent()
|
||||
} catch (error) {
|
||||
if (error.code === 'E_ROW_NOT_FOUND') {
|
||||
return response.notFound({
|
||||
error: 'Keyword not found',
|
||||
})
|
||||
}
|
||||
|
||||
return response.internalServerError({
|
||||
error: 'Failed to delete keyword',
|
||||
message: error instanceof Error ? error.message : 'Unknown error',
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import ParseLog from '#models/parse_log'
|
||||
|
||||
export default class ParseLogsController {
|
||||
/**
|
||||
* List parse logs with pagination, ordered by latest
|
||||
* GET /api/parse-logs?page=1&limit=20
|
||||
*/
|
||||
async index({ request, response }: HttpContext) {
|
||||
const page = request.input('page', 1)
|
||||
const limit = request.input('limit', 20)
|
||||
|
||||
const pageNumber = Math.max(1, Number(page))
|
||||
const limitNumber = Math.min(100, Math.max(1, Number(limit)))
|
||||
|
||||
try {
|
||||
const parseLogs = await ParseLog.query()
|
||||
.orderBy('started_at', 'desc')
|
||||
.paginate(pageNumber, limitNumber)
|
||||
|
||||
return response.ok(parseLogs.serialize())
|
||||
} catch (error) {
|
||||
return response.internalServerError({
|
||||
error: 'Failed to fetch parse logs',
|
||||
message: error instanceof Error ? error.message : 'Unknown error',
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import app from '@adonisjs/core/services/app'
|
||||
import { HttpContext, ExceptionHandler } from '@adonisjs/core/http'
|
||||
import type { StatusPageRange, StatusPageRenderer } from '@adonisjs/core/types/http'
|
||||
|
||||
export default class HttpExceptionHandler extends ExceptionHandler {
|
||||
/**
|
||||
* In debug mode, the exception handler will display verbose errors
|
||||
* with pretty printed stack traces.
|
||||
*/
|
||||
protected debug = !app.inProduction
|
||||
|
||||
/**
|
||||
* Status pages are used to display a custom HTML pages for certain error
|
||||
* codes. You might want to enable them in production only, but feel
|
||||
* free to enable them in development as well.
|
||||
*/
|
||||
protected renderStatusPages = app.inProduction
|
||||
|
||||
/**
|
||||
* Status pages is a collection of error code range and a callback
|
||||
* to return the HTML contents to send as a response.
|
||||
*/
|
||||
protected statusPages: Record<StatusPageRange, StatusPageRenderer> = {
|
||||
'404': (error, { view }) => {
|
||||
return view.render('pages/errors/not_found', { error })
|
||||
},
|
||||
'500..599': (error, { view }) => {
|
||||
return view.render('pages/errors/server_error', { error })
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* The method is used for handling errors and returning
|
||||
* response to the client
|
||||
*/
|
||||
async handle(error: unknown, ctx: HttpContext) {
|
||||
return super.handle(error, ctx)
|
||||
}
|
||||
|
||||
/**
|
||||
* The method is used to report error to the logging service or
|
||||
* the a third party error monitoring service.
|
||||
*
|
||||
* @note You should not attempt to send a response from this method.
|
||||
*/
|
||||
async report(error: unknown, ctx: HttpContext) {
|
||||
return super.report(error, ctx)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import type { NextFn } from '@adonisjs/core/types/http'
|
||||
import type { Authenticators } from '@adonisjs/auth/types'
|
||||
|
||||
/**
|
||||
* Auth middleware is used authenticate HTTP requests and deny
|
||||
* access to unauthenticated users.
|
||||
*/
|
||||
export default class AuthMiddleware {
|
||||
/**
|
||||
* The URL to redirect to, when authentication fails
|
||||
*/
|
||||
redirectTo = '/login'
|
||||
|
||||
async handle(
|
||||
ctx: HttpContext,
|
||||
next: NextFn,
|
||||
options: {
|
||||
guards?: (keyof Authenticators)[]
|
||||
} = {}
|
||||
) {
|
||||
await ctx.auth.authenticateUsing(options.guards, { loginRoute: this.redirectTo })
|
||||
return next()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { Logger } from '@adonisjs/core/logger'
|
||||
import { HttpContext } from '@adonisjs/core/http'
|
||||
import { NextFn } from '@adonisjs/core/types/http'
|
||||
|
||||
/**
|
||||
* The container bindings middleware binds classes to their request
|
||||
* specific value using the container resolver.
|
||||
*
|
||||
* - We bind "HttpContext" class to the "ctx" object
|
||||
* - And bind "Logger" class to the "ctx.logger" object
|
||||
*/
|
||||
export default class ContainerBindingsMiddleware {
|
||||
handle(ctx: HttpContext, next: NextFn) {
|
||||
ctx.containerResolver.bindValue(HttpContext, ctx)
|
||||
ctx.containerResolver.bindValue(Logger, ctx.logger)
|
||||
|
||||
return next()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import type { NextFn } from '@adonisjs/core/types/http'
|
||||
import type { Authenticators } from '@adonisjs/auth/types'
|
||||
|
||||
/**
|
||||
* Guest middleware is used to deny access to routes that should
|
||||
* be accessed by unauthenticated users.
|
||||
*
|
||||
* For example, the login page should not be accessible if the user
|
||||
* is already logged-in
|
||||
*/
|
||||
export default class GuestMiddleware {
|
||||
/**
|
||||
* The URL to redirect to when user is logged-in
|
||||
*/
|
||||
redirectTo = '/'
|
||||
|
||||
async handle(
|
||||
ctx: HttpContext,
|
||||
next: NextFn,
|
||||
options: { guards?: (keyof Authenticators)[] } = {}
|
||||
) {
|
||||
for (let guard of options.guards || [ctx.auth.defaultGuard]) {
|
||||
if (await ctx.auth.use(guard).check()) {
|
||||
return ctx.response.redirect(this.redirectTo, true)
|
||||
}
|
||||
}
|
||||
|
||||
return next()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import type { NextFn } from '@adonisjs/core/types/http'
|
||||
|
||||
/**
|
||||
* Silent auth middleware can be used as a global middleware to silent check
|
||||
* if the user is logged-in or not.
|
||||
*
|
||||
* The request continues as usual, even when the user is not logged-in.
|
||||
*/
|
||||
export default class SilentAuthMiddleware {
|
||||
async handle(
|
||||
ctx: HttpContext,
|
||||
next: NextFn,
|
||||
) {
|
||||
await ctx.auth.check()
|
||||
|
||||
return next()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { DateTime } from 'luxon'
|
||||
import { BaseModel, column, hasMany } from '@adonisjs/lucid/orm'
|
||||
import type { HasMany } from '@adonisjs/lucid/types/relations'
|
||||
import Notification from '#models/notification'
|
||||
|
||||
export default class Auction extends BaseModel {
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare auctionNum: string
|
||||
|
||||
@column()
|
||||
declare title: string
|
||||
|
||||
@column()
|
||||
declare description: string | null
|
||||
|
||||
@column()
|
||||
declare organization: string | null
|
||||
|
||||
@column()
|
||||
declare status: string | null
|
||||
|
||||
@column()
|
||||
declare price: number | null
|
||||
|
||||
@column.dateTime()
|
||||
declare deadline: DateTime | null
|
||||
|
||||
@column()
|
||||
declare url: string | null
|
||||
|
||||
@column({
|
||||
prepare: (value: Record<string, any> | null) => (value ? JSON.stringify(value) : null),
|
||||
consume: (value: string | null) => (value ? JSON.parse(value) : null),
|
||||
})
|
||||
declare rawData: Record<string, any> | null
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime
|
||||
|
||||
@hasMany(() => Notification)
|
||||
declare notifications: HasMany<typeof Notification>
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { DateTime } from 'luxon'
|
||||
import { BaseModel, column, belongsTo } from '@adonisjs/lucid/orm'
|
||||
import type { BelongsTo } from '@adonisjs/lucid/types/relations'
|
||||
import User from '#models/user'
|
||||
|
||||
export default class Keyword extends BaseModel {
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare userId: number
|
||||
|
||||
@column()
|
||||
declare keyword: string
|
||||
|
||||
@column()
|
||||
declare isActive: boolean
|
||||
|
||||
@column()
|
||||
declare caseSensitive: boolean
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime
|
||||
|
||||
@belongsTo(() => User)
|
||||
declare user: BelongsTo<typeof User>
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { DateTime } from 'luxon'
|
||||
import { BaseModel, column, belongsTo } from '@adonisjs/lucid/orm'
|
||||
import type { BelongsTo } from '@adonisjs/lucid/types/relations'
|
||||
import Auction from '#models/auction'
|
||||
import Keyword from '#models/keyword'
|
||||
|
||||
export default class Notification extends BaseModel {
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare auctionId: number
|
||||
|
||||
@column()
|
||||
declare keywordId: number
|
||||
|
||||
@column()
|
||||
declare status: 'pending' | 'sent' | 'failed'
|
||||
|
||||
@column()
|
||||
declare errorMessage: string | null
|
||||
|
||||
@column.dateTime()
|
||||
declare sentAt: DateTime | null
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime
|
||||
|
||||
@belongsTo(() => Auction)
|
||||
declare auction: BelongsTo<typeof Auction>
|
||||
|
||||
@belongsTo(() => Keyword)
|
||||
declare keyword: BelongsTo<typeof Keyword>
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { DateTime } from 'luxon'
|
||||
import { BaseModel, column } from '@adonisjs/lucid/orm'
|
||||
|
||||
export default class ParseLog extends BaseModel {
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare parseType: string
|
||||
|
||||
@column()
|
||||
declare status: string
|
||||
|
||||
@column()
|
||||
declare itemsFound: number
|
||||
|
||||
@column()
|
||||
declare errors: string | null
|
||||
|
||||
@column.dateTime()
|
||||
declare startedAt: DateTime
|
||||
|
||||
@column.dateTime()
|
||||
declare completedAt: DateTime | null
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { DateTime } from 'luxon'
|
||||
import hash from '@adonisjs/core/services/hash'
|
||||
import { compose } from '@adonisjs/core/helpers'
|
||||
import { BaseModel, column } from '@adonisjs/lucid/orm'
|
||||
import { withAuthFinder } from '@adonisjs/auth/mixins/lucid'
|
||||
|
||||
const AuthFinder = withAuthFinder(() => hash.use('scrypt'), {
|
||||
uids: ['email'],
|
||||
passwordColumnName: 'password',
|
||||
})
|
||||
|
||||
export default class User extends compose(BaseModel, AuthFinder) {
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare email: string
|
||||
|
||||
@column({ serializeAs: null })
|
||||
declare password: string
|
||||
|
||||
@column()
|
||||
declare telegramChatId: string | null
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
/**
|
||||
* Zod schema for auction data validation
|
||||
* Ensures all scraped data conforms to expected structure before database insertion
|
||||
*/
|
||||
export const AuctionSchema = z.object({
|
||||
auctionNum: z.string().trim().min(1, 'Auction number is required'),
|
||||
title: z.string().trim().min(1, 'Title is required'),
|
||||
organization: z.string().trim().min(1, 'Organization is required'),
|
||||
status: z.string().trim().min(1, 'Status is required'),
|
||||
deadline: z.string().nullable().default(null),
|
||||
link: z.string().url('Link must be a valid URL'),
|
||||
description: z.string().nullable().default(null),
|
||||
})
|
||||
|
||||
/**
|
||||
* TypeScript type inferred from schema
|
||||
*/
|
||||
export type AuctionData = z.infer<typeof AuctionSchema>
|
||||
@@ -0,0 +1,223 @@
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { DateTime } from 'luxon'
|
||||
import Auction from '#models/auction'
|
||||
import Keyword from '#models/keyword'
|
||||
import Notification from '#models/notification'
|
||||
import User from '#models/user'
|
||||
import { TelegramService } from '#services/telegram_service'
|
||||
|
||||
/**
|
||||
* NotificationService - Handles keyword matching and notification delivery
|
||||
*
|
||||
* Features:
|
||||
* - Matches keywords against auction titles and descriptions
|
||||
* - Case-sensitive and case-insensitive matching support
|
||||
* - Prevents duplicate notifications (checks existing records)
|
||||
* - Tracks notification status (pending/sent/failed)
|
||||
* - Integrates with TelegramService for message delivery
|
||||
* - Validates user active status before sending
|
||||
*/
|
||||
export class NotificationService {
|
||||
/**
|
||||
* Check auction against all active keywords and send notifications for matches
|
||||
*
|
||||
* @param auction - The auction to check for keyword matches
|
||||
*/
|
||||
async checkAndNotify(auction: Auction): Promise<void> {
|
||||
try {
|
||||
logger.info(`Checking auction ${auction.auctionNum} for keyword matches`)
|
||||
|
||||
// Get all active keywords
|
||||
const keywords = await Keyword.query().where('isActive', true).preload('user')
|
||||
|
||||
if (keywords.length === 0) {
|
||||
logger.debug('No active keywords found, skipping notification check')
|
||||
return
|
||||
}
|
||||
|
||||
logger.debug(`Found ${keywords.length} active keyword(s) to check`)
|
||||
|
||||
let matchCount = 0
|
||||
|
||||
for (const keyword of keywords) {
|
||||
try {
|
||||
// Check if keyword matches auction title or description
|
||||
const titleMatch = this.matchKeyword(
|
||||
auction.title,
|
||||
keyword.keyword,
|
||||
keyword.caseSensitive
|
||||
)
|
||||
|
||||
const descMatch = auction.description
|
||||
? this.matchKeyword(auction.description, keyword.keyword, keyword.caseSensitive)
|
||||
: false
|
||||
|
||||
if (!titleMatch && !descMatch) {
|
||||
continue // No match, skip this keyword
|
||||
}
|
||||
|
||||
matchCount++
|
||||
logger.info(
|
||||
`Keyword match found: "${keyword.keyword}" in auction ${auction.auctionNum}`,
|
||||
{
|
||||
keywordId: keyword.id,
|
||||
auctionId: auction.id,
|
||||
matchedIn: titleMatch ? 'title' : 'description',
|
||||
}
|
||||
)
|
||||
|
||||
// Check if notification already exists (prevent duplicates)
|
||||
const existingNotification = await Notification.query()
|
||||
.where('auctionId', auction.id)
|
||||
.where('keywordId', keyword.id)
|
||||
.first()
|
||||
|
||||
if (existingNotification) {
|
||||
logger.debug(
|
||||
`Notification already exists for auction ${auction.id} and keyword ${keyword.id}, skipping`,
|
||||
{
|
||||
notificationId: existingNotification.id,
|
||||
status: existingNotification.status,
|
||||
}
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
// Create notification record with 'pending' status
|
||||
const notification = await Notification.create({
|
||||
auctionId: auction.id,
|
||||
keywordId: keyword.id,
|
||||
status: 'pending',
|
||||
errorMessage: null,
|
||||
sentAt: null,
|
||||
})
|
||||
|
||||
logger.debug(`Created notification record ${notification.id}`)
|
||||
|
||||
// Get user and validate active status
|
||||
const user = await User.find(keyword.userId)
|
||||
|
||||
if (!user) {
|
||||
logger.warn(`User ${keyword.userId} not found for keyword ${keyword.id}`)
|
||||
await notification
|
||||
.merge({
|
||||
status: 'failed',
|
||||
errorMessage: 'User not found',
|
||||
})
|
||||
.save()
|
||||
continue
|
||||
}
|
||||
|
||||
if (!user.telegramChatId) {
|
||||
logger.warn(`User ${user.id} has no Telegram chat ID`)
|
||||
await notification
|
||||
.merge({
|
||||
status: 'failed',
|
||||
errorMessage: 'User has no Telegram chat ID',
|
||||
})
|
||||
.save()
|
||||
continue
|
||||
}
|
||||
|
||||
// Send notification via Telegram
|
||||
try {
|
||||
const telegramService = TelegramService.getInstance()
|
||||
const auctionUrl = auction.url || `https://icetrade.by/auction/${auction.auctionNum}`
|
||||
|
||||
const success = await telegramService.sendNotification(
|
||||
user.telegramChatId,
|
||||
auction.title,
|
||||
auctionUrl,
|
||||
keyword.keyword
|
||||
)
|
||||
|
||||
if (success) {
|
||||
// Update notification status to 'sent'
|
||||
await notification
|
||||
.merge({
|
||||
status: 'sent',
|
||||
sentAt: DateTime.now(),
|
||||
})
|
||||
.save()
|
||||
|
||||
logger.info(`Notification sent successfully`, {
|
||||
notificationId: notification.id,
|
||||
userId: user.id,
|
||||
chatId: user.telegramChatId,
|
||||
})
|
||||
} else {
|
||||
// Update notification status to 'failed'
|
||||
await notification
|
||||
.merge({
|
||||
status: 'failed',
|
||||
errorMessage: 'Failed to send Telegram message (unknown error)',
|
||||
})
|
||||
.save()
|
||||
|
||||
logger.error(`Failed to send notification`, {
|
||||
notificationId: notification.id,
|
||||
userId: user.id,
|
||||
chatId: user.telegramChatId,
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : 'Unknown error during notification send'
|
||||
|
||||
logger.error(`Error sending notification`, {
|
||||
notificationId: notification.id,
|
||||
error: errorMessage,
|
||||
userId: user.id,
|
||||
})
|
||||
|
||||
// Update notification status to 'failed' with error details
|
||||
await notification
|
||||
.merge({
|
||||
status: 'failed',
|
||||
errorMessage,
|
||||
})
|
||||
.save()
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Error processing keyword ${keyword.id} for auction ${auction.id}`, {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
keywordId: keyword.id,
|
||||
auctionId: auction.id,
|
||||
})
|
||||
// Continue processing other keywords even if one fails
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Completed notification check for auction ${auction.auctionNum}: ${matchCount} match(es) found`
|
||||
)
|
||||
} catch (error) {
|
||||
logger.error(`Error in checkAndNotify for auction ${auction.id}`, {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
auctionId: auction.id,
|
||||
})
|
||||
// Don't throw - allow scraper to continue processing other auctions
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a keyword matches text with case-sensitive or case-insensitive matching
|
||||
*
|
||||
* @param text - The text to search in
|
||||
* @param keyword - The keyword to search for
|
||||
* @param caseSensitive - Whether to perform case-sensitive matching
|
||||
* @returns true if keyword is found in text, false otherwise
|
||||
*/
|
||||
private matchKeyword(text: string, keyword: string, caseSensitive: boolean): boolean {
|
||||
if (!text || !keyword) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (caseSensitive) {
|
||||
return text.includes(keyword)
|
||||
}
|
||||
|
||||
return text.toLowerCase().includes(keyword.toLowerCase())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
import * as cheerio from 'cheerio'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { AuctionSchema, type AuctionData } from '../schemas/auction_schema.js'
|
||||
|
||||
/**
|
||||
* Configuration for the scraper
|
||||
*/
|
||||
const SCRAPER_CONFIG = {
|
||||
baseUrl: 'https://icetrade.by',
|
||||
requestDelay: 1000, // 1 second between requests
|
||||
timeout: 30000, // 30 seconds timeout
|
||||
userAgent:
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
maxRetries: 3,
|
||||
retryDelay: 2000,
|
||||
} as const
|
||||
|
||||
/**
|
||||
* Error thrown when scraping fails
|
||||
*/
|
||||
export class ScraperError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly cause?: unknown
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'ScraperError'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* ScraperService - Handles fetching and parsing auction data from icetrade.by
|
||||
*
|
||||
* Features:
|
||||
* - Rate limiting (1s delay between requests)
|
||||
* - Retry logic with exponential backoff
|
||||
* - Comprehensive error handling
|
||||
* - Data validation with Zod
|
||||
* - Structured logging
|
||||
*/
|
||||
export class ScraperService {
|
||||
/**
|
||||
* Builds the URL for fetching auctions with all required parameters
|
||||
*/
|
||||
private buildUrl(pageNumber: number): string {
|
||||
const params = new URLSearchParams({
|
||||
search_text: '',
|
||||
'zakup_type[1]': '1',
|
||||
'zakup_type[2]': '1',
|
||||
onPage: '100',
|
||||
sort: 'num:desc',
|
||||
p: pageNumber.toString(),
|
||||
// Status flags
|
||||
'r[1]': '1',
|
||||
'r[2]': '1',
|
||||
'r[3]': '1',
|
||||
'r[4]': '1',
|
||||
'r[5]': '1',
|
||||
'r[6]': '1',
|
||||
'r[7]': '1',
|
||||
// Trade type flags
|
||||
't[Trade]': '1',
|
||||
't[contest]': '1',
|
||||
't[request]': '1',
|
||||
't[qualification]': '1',
|
||||
't[negotiations]': '1',
|
||||
})
|
||||
|
||||
return `${SCRAPER_CONFIG.baseUrl}/trades/index?${params.toString()}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches HTML content from the specified page with retry logic
|
||||
*/
|
||||
async fetchPage(pageNumber: number): Promise<string> {
|
||||
const url = this.buildUrl(pageNumber)
|
||||
let lastError: Error | undefined
|
||||
|
||||
logger.info(`Preparing to fetch URL: ${url}`)
|
||||
|
||||
for (let attempt = 1; attempt <= SCRAPER_CONFIG.maxRetries; attempt++) {
|
||||
try {
|
||||
logger.info(
|
||||
`Fetching page ${pageNumber} (attempt ${attempt}/${SCRAPER_CONFIG.maxRetries})`,
|
||||
{ url }
|
||||
)
|
||||
|
||||
const controller = new AbortController()
|
||||
const timeoutId = setTimeout(() => controller.abort(), SCRAPER_CONFIG.timeout)
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': SCRAPER_CONFIG.userAgent,
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
Connection: 'keep-alive',
|
||||
},
|
||||
signal: controller.signal,
|
||||
})
|
||||
|
||||
clearTimeout(timeoutId)
|
||||
|
||||
logger.info(`Received response: status=${response.status} ${response.statusText}`)
|
||||
|
||||
if (!response.ok) {
|
||||
const bodyText = await response.text().catch(() => 'Unable to read response body')
|
||||
throw new Error(
|
||||
`HTTP ${response.status}: ${response.statusText}. Body: ${bodyText.substring(0, 200)}`
|
||||
)
|
||||
}
|
||||
|
||||
const html = await response.text()
|
||||
|
||||
if (!html || html.trim().length === 0) {
|
||||
throw new Error('Received empty response')
|
||||
}
|
||||
|
||||
logger.info(`Successfully fetched page ${pageNumber}: ${html.length} bytes`)
|
||||
return html
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error))
|
||||
|
||||
// Log detailed error information
|
||||
const errorDetails: Record<string, any> = {
|
||||
message: lastError.message,
|
||||
name: lastError.name,
|
||||
url,
|
||||
}
|
||||
|
||||
// Add stack trace for non-HTTP errors
|
||||
if (!(lastError.message.startsWith('HTTP '))) {
|
||||
errorDetails.stack = lastError.stack
|
||||
}
|
||||
|
||||
// Check for specific error types
|
||||
if (lastError.name === 'AbortError') {
|
||||
errorDetails.reason = 'Request timeout after 30s'
|
||||
} else if (lastError.message.includes('fetch failed')) {
|
||||
errorDetails.reason = 'Network error - check DNS, firewall, or connectivity'
|
||||
} else if (lastError.message.includes('ENOTFOUND')) {
|
||||
errorDetails.reason = 'DNS resolution failed - domain not found'
|
||||
} else if (lastError.message.includes('ECONNREFUSED')) {
|
||||
errorDetails.reason = 'Connection refused - server not reachable'
|
||||
} else if (lastError.message.includes('ETIMEDOUT')) {
|
||||
errorDetails.reason = 'Connection timeout - server too slow or unreachable'
|
||||
}
|
||||
|
||||
logger.warn(errorDetails, `Failed to fetch page ${pageNumber} (attempt ${attempt}/${SCRAPER_CONFIG.maxRetries})`)
|
||||
|
||||
if (attempt < SCRAPER_CONFIG.maxRetries) {
|
||||
const delay = SCRAPER_CONFIG.retryDelay * attempt
|
||||
logger.info(`Retrying in ${delay}ms...`)
|
||||
await this.delay(delay)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const finalError = new ScraperError(
|
||||
`Failed to fetch page ${pageNumber} after ${SCRAPER_CONFIG.maxRetries} attempts: ${lastError?.message}`,
|
||||
lastError
|
||||
)
|
||||
|
||||
logger.error('All fetch attempts failed', {
|
||||
pageNumber,
|
||||
url,
|
||||
lastErrorMessage: lastError?.message,
|
||||
lastErrorName: lastError?.name,
|
||||
})
|
||||
|
||||
throw finalError
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses HTML content and extracts auction data
|
||||
*/
|
||||
parsePage(html: string): AuctionData[] {
|
||||
try {
|
||||
const $ = cheerio.load(html)
|
||||
const auctions: AuctionData[] = []
|
||||
|
||||
// Find the auctions table
|
||||
const auctionsTable = $('table.auctions.w100')
|
||||
|
||||
if (auctionsTable.length === 0) {
|
||||
logger.warn('No auctions table found in HTML')
|
||||
return []
|
||||
}
|
||||
|
||||
// Parse each auction row
|
||||
const rows = auctionsTable.find('tbody tr')
|
||||
logger.info(`Found ${rows.length} auction rows to parse`)
|
||||
|
||||
rows.each((index, element) => {
|
||||
try {
|
||||
const row = $(element)
|
||||
|
||||
// Extract auction data from table cells
|
||||
const cells = row.find('td')
|
||||
|
||||
if (cells.length < 4) {
|
||||
logger.warn(`Row ${index} has insufficient cells, skipping`)
|
||||
return
|
||||
}
|
||||
|
||||
// Extract auction number (typically in first cell)
|
||||
const auctionNumCell = $(cells[0])
|
||||
const auctionNum = auctionNumCell.text().trim()
|
||||
|
||||
// Extract title and link (typically in second cell with <a> tag)
|
||||
const titleCell = $(cells[1])
|
||||
const titleLink = titleCell.find('a').first()
|
||||
const title = titleLink.text().trim()
|
||||
const link = titleLink.attr('href')?.trim() || ''
|
||||
|
||||
// Skip if link is empty or missing
|
||||
if (!link) {
|
||||
logger.warn(`Row ${index} has missing or empty link, skipping`)
|
||||
return
|
||||
}
|
||||
|
||||
// Make link absolute if it's relative
|
||||
const absoluteLink = link.startsWith('http')
|
||||
? link
|
||||
: `${SCRAPER_CONFIG.baseUrl}${link.startsWith('/') ? link : `/${link}`}`
|
||||
|
||||
// Extract organization (typically in third cell)
|
||||
const organizationCell = $(cells[2])
|
||||
const organization = organizationCell.text().trim()
|
||||
|
||||
// Extract status (typically in fourth cell)
|
||||
const statusCell = $(cells[3])
|
||||
const status = statusCell.text().trim()
|
||||
|
||||
// Extract deadline if available (typically in fifth cell)
|
||||
const deadlineCell = $(cells[4])
|
||||
const deadline = deadlineCell.text().trim() || null
|
||||
|
||||
// Description can be extracted from title cell's additional text or separate element
|
||||
const description = titleCell.find('.description').text().trim() || null
|
||||
|
||||
// Validate with Zod schema
|
||||
const result = AuctionSchema.safeParse({
|
||||
auctionNum,
|
||||
title,
|
||||
organization,
|
||||
status,
|
||||
deadline,
|
||||
link: absoluteLink,
|
||||
description,
|
||||
})
|
||||
|
||||
if (!result.success) {
|
||||
logger.warn(`Validation failed for auction at row ${index}`, {
|
||||
errors: result.error.issues,
|
||||
data: { auctionNum, title },
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
auctions.push(result.data)
|
||||
} catch (error) {
|
||||
logger.error(`Error parsing auction row ${index}`, {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
logger.info(`Successfully parsed ${auctions.length} valid auctions`)
|
||||
return auctions
|
||||
} catch (error) {
|
||||
throw new ScraperError(
|
||||
'Failed to parse HTML content',
|
||||
error instanceof Error ? error : new Error(String(error))
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrapes multiple pages of auctions with rate limiting
|
||||
*
|
||||
* @param maxPages - Maximum number of pages to scrape (default: 1)
|
||||
* @returns Array of all parsed auction data
|
||||
*/
|
||||
async scrapeAuctions(maxPages: number = 1): Promise<AuctionData[]> {
|
||||
if (maxPages < 1) {
|
||||
throw new Error('maxPages must be at least 1')
|
||||
}
|
||||
|
||||
logger.info(`Starting scrape of ${maxPages} page(s)`)
|
||||
const allAuctions: AuctionData[] = []
|
||||
|
||||
for (let page = 1; page <= maxPages; page++) {
|
||||
try {
|
||||
// Fetch page HTML
|
||||
const html = await this.fetchPage(page)
|
||||
|
||||
// Parse auctions from HTML
|
||||
const auctions = this.parsePage(html)
|
||||
|
||||
allAuctions.push(...auctions)
|
||||
|
||||
logger.info(`Page ${page}/${maxPages}: Found ${auctions.length} auctions`)
|
||||
|
||||
// Rate limiting: wait before next request (except for last page)
|
||||
if (page < maxPages) {
|
||||
logger.debug(`Waiting ${SCRAPER_CONFIG.requestDelay}ms before next request`)
|
||||
await this.delay(SCRAPER_CONFIG.requestDelay)
|
||||
}
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error))
|
||||
logger.error(`Error scraping page ${page}`, {
|
||||
message: err.message,
|
||||
name: err.name,
|
||||
stack: err.stack,
|
||||
cause: err.cause,
|
||||
})
|
||||
|
||||
// Continue to next page instead of failing completely
|
||||
// This ensures partial data is still returned
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Scraping completed: ${allAuctions.length} total auctions from ${maxPages} page(s)`
|
||||
)
|
||||
|
||||
return allAuctions
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method for delays (rate limiting, retries)
|
||||
*/
|
||||
private delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
import { Bot, Context } from 'grammy'
|
||||
import env from '#start/env'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import User from '#models/user'
|
||||
import Keyword from '#models/keyword'
|
||||
|
||||
/**
|
||||
* TelegramService - Handles Telegram bot operations
|
||||
*
|
||||
* Singleton service for managing Grammy bot instance and commands.
|
||||
* Provides keyword management and notification delivery.
|
||||
*/
|
||||
export class TelegramService {
|
||||
private static instance: TelegramService | null = null
|
||||
private bot: Bot
|
||||
private isRunning: boolean = false
|
||||
|
||||
private constructor() {
|
||||
const token = env.get('TELEGRAM_BOT_TOKEN')
|
||||
this.bot = new Bot(token)
|
||||
this.setupCommands()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get singleton instance of TelegramService
|
||||
*/
|
||||
static getInstance(): TelegramService {
|
||||
if (!TelegramService.instance) {
|
||||
TelegramService.instance = new TelegramService()
|
||||
}
|
||||
return TelegramService.instance
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup bot commands and handlers
|
||||
*/
|
||||
private setupCommands(): void {
|
||||
// /start - Register user
|
||||
this.bot.command('start', async (ctx) => {
|
||||
try {
|
||||
await this.handleStart(ctx)
|
||||
} catch (error) {
|
||||
logger.error('Error handling /start command', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
chatId: ctx.chat?.id,
|
||||
})
|
||||
await ctx.reply('Произошла ошибка при регистрации. Попробуйте позже.')
|
||||
}
|
||||
})
|
||||
|
||||
// /addkeyword - Add keyword
|
||||
this.bot.command('addkeyword', async (ctx) => {
|
||||
try {
|
||||
await this.handleAddKeyword(ctx)
|
||||
} catch (error) {
|
||||
logger.error('Error handling /addkeyword command', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
chatId: ctx.chat?.id,
|
||||
})
|
||||
await ctx.reply('Произошла ошибка при добавлении ключевого слова. Попробуйте позже.')
|
||||
}
|
||||
})
|
||||
|
||||
// /listkeywords - List keywords
|
||||
this.bot.command('listkeywords', async (ctx) => {
|
||||
try {
|
||||
await this.handleListKeywords(ctx)
|
||||
} catch (error) {
|
||||
logger.error('Error handling /listkeywords command', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
chatId: ctx.chat?.id,
|
||||
})
|
||||
await ctx.reply('Произошла ошибка при получении списка ключевых слов. Попробуйте позже.')
|
||||
}
|
||||
})
|
||||
|
||||
// /deletekeyword - Delete keyword
|
||||
this.bot.command('deletekeyword', async (ctx) => {
|
||||
try {
|
||||
await this.handleDeleteKeyword(ctx)
|
||||
} catch (error) {
|
||||
logger.error('Error handling /deletekeyword command', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
chatId: ctx.chat?.id,
|
||||
})
|
||||
await ctx.reply('Произошла ошибка при удалении ключевого слова. Попробуйте позже.')
|
||||
}
|
||||
})
|
||||
|
||||
// /help - Show help
|
||||
this.bot.command('help', async (ctx) => {
|
||||
await this.handleHelp(ctx)
|
||||
})
|
||||
|
||||
// Error handler
|
||||
this.bot.catch((err) => {
|
||||
logger.error('Grammy bot error', {
|
||||
error: err.error instanceof Error ? err.error.message : String(err.error),
|
||||
ctx: err.ctx,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle /start command - Register user
|
||||
*/
|
||||
private async handleStart(ctx: Context): Promise<void> {
|
||||
if (!ctx.chat?.id) {
|
||||
await ctx.reply('Не удалось определить chat ID.')
|
||||
return
|
||||
}
|
||||
|
||||
const chatId = String(ctx.chat.id)
|
||||
|
||||
// Check if user already exists
|
||||
const existingUser = await User.findBy('telegramChatId', chatId)
|
||||
|
||||
if (existingUser) {
|
||||
await ctx.reply(
|
||||
'Вы уже зарегистрированы!\n\n' +
|
||||
'Используйте /addkeyword для добавления ключевых слов.\n' +
|
||||
'Используйте /help для получения справки.'
|
||||
)
|
||||
logger.info(`User already registered: ${chatId}`)
|
||||
return
|
||||
}
|
||||
|
||||
// Create new user
|
||||
const user = await User.create({
|
||||
email: `telegram_${chatId}@temp.local`,
|
||||
password: Math.random().toString(36).substring(2, 15),
|
||||
telegramChatId: chatId,
|
||||
})
|
||||
|
||||
await ctx.reply(
|
||||
'Добро пожаловать! Вы успешно зарегистрированы.\n\n' +
|
||||
'Теперь вы можете добавлять ключевые слова для отслеживания аукционов:\n' +
|
||||
'/addkeyword <слово> - добавить ключевое слово\n' +
|
||||
'/listkeywords - список ваших ключевых слов\n' +
|
||||
'/deletekeyword <id> - удалить ключевое слово\n' +
|
||||
'/help - справка'
|
||||
)
|
||||
|
||||
logger.info(`New user registered: ${chatId} (user_id: ${user.id})`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle /addkeyword command - Add keyword for user
|
||||
*/
|
||||
private async handleAddKeyword(ctx: Context): Promise<void> {
|
||||
if (!ctx.chat?.id) {
|
||||
await ctx.reply('Не удалось определить chat ID.')
|
||||
return
|
||||
}
|
||||
|
||||
const chatId = String(ctx.chat.id)
|
||||
const user = await User.findBy('telegramChatId', chatId)
|
||||
|
||||
if (!user) {
|
||||
await ctx.reply('Вы не зарегистрированы. Используйте /start для регистрации.')
|
||||
return
|
||||
}
|
||||
|
||||
// Extract keyword from message
|
||||
const messageText = ctx.message?.text || ''
|
||||
const parts = messageText.split(' ')
|
||||
|
||||
if (parts.length < 2) {
|
||||
await ctx.reply(
|
||||
'Использование: /addkeyword <слово>\n\n' + 'Пример: /addkeyword строительство'
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
const keyword = parts.slice(1).join(' ').trim()
|
||||
|
||||
if (keyword.length === 0) {
|
||||
await ctx.reply('Ключевое слово не может быть пустым.')
|
||||
return
|
||||
}
|
||||
|
||||
if (keyword.length > 255) {
|
||||
await ctx.reply('Ключевое слово слишком длинное (максимум 255 символов).')
|
||||
return
|
||||
}
|
||||
|
||||
// Check if keyword already exists for this user
|
||||
const existingKeyword = await Keyword.query()
|
||||
.where('userId', user.id)
|
||||
.where('keyword', keyword)
|
||||
.first()
|
||||
|
||||
if (existingKeyword) {
|
||||
await ctx.reply(`Ключевое слово "${keyword}" уже добавлено.`)
|
||||
return
|
||||
}
|
||||
|
||||
// Create keyword
|
||||
const newKeyword = await Keyword.create({
|
||||
userId: user.id,
|
||||
keyword: keyword,
|
||||
isActive: true,
|
||||
})
|
||||
|
||||
await ctx.reply(`Ключевое слово "${keyword}" успешно добавлено (ID: ${newKeyword.id}).`)
|
||||
|
||||
logger.info(`Keyword added: "${keyword}" for user ${user.id} (chat_id: ${chatId})`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle /listkeywords command - List user's keywords
|
||||
*/
|
||||
private async handleListKeywords(ctx: Context): Promise<void> {
|
||||
if (!ctx.chat?.id) {
|
||||
await ctx.reply('Не удалось определить chat ID.')
|
||||
return
|
||||
}
|
||||
|
||||
const chatId = String(ctx.chat.id)
|
||||
const user = await User.findBy('telegramChatId', chatId)
|
||||
|
||||
if (!user) {
|
||||
await ctx.reply('Вы не зарегистрированы. Используйте /start для регистрации.')
|
||||
return
|
||||
}
|
||||
|
||||
// Fetch user's keywords
|
||||
const keywords = await Keyword.query().where('userId', user.id).where('isActive', true)
|
||||
|
||||
if (keywords.length === 0) {
|
||||
await ctx.reply(
|
||||
'У вас нет активных ключевых слов.\n\n' +
|
||||
'Используйте /addkeyword для добавления ключевых слов.'
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
// Format keywords list
|
||||
const keywordsList = keywords
|
||||
.map((kw) => `${kw.id}. ${kw.keyword}`)
|
||||
.join('\n')
|
||||
|
||||
await ctx.reply(`Ваши ключевые слова:\n\n${keywordsList}\n\nДля удаления используйте: /deletekeyword <id>`)
|
||||
|
||||
logger.info(`Listed ${keywords.length} keyword(s) for user ${user.id} (chat_id: ${chatId})`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle /deletekeyword command - Delete keyword by ID
|
||||
*/
|
||||
private async handleDeleteKeyword(ctx: Context): Promise<void> {
|
||||
if (!ctx.chat?.id) {
|
||||
await ctx.reply('Не удалось определить chat ID.')
|
||||
return
|
||||
}
|
||||
|
||||
const chatId = String(ctx.chat.id)
|
||||
const user = await User.findBy('telegramChatId', chatId)
|
||||
|
||||
if (!user) {
|
||||
await ctx.reply('Вы не зарегистрированы. Используйте /start для регистрации.')
|
||||
return
|
||||
}
|
||||
|
||||
// Extract keyword ID from message
|
||||
const messageText = ctx.message?.text || ''
|
||||
const parts = messageText.split(' ')
|
||||
|
||||
if (parts.length < 2) {
|
||||
await ctx.reply('Использование: /deletekeyword <id>\n\n' + 'Пример: /deletekeyword 5')
|
||||
return
|
||||
}
|
||||
|
||||
const keywordId = Number.parseInt(parts[1], 10)
|
||||
|
||||
if (Number.isNaN(keywordId)) {
|
||||
await ctx.reply('Некорректный ID. Используйте числовое значение.')
|
||||
return
|
||||
}
|
||||
|
||||
// Find keyword
|
||||
const keyword = await Keyword.query()
|
||||
.where('id', keywordId)
|
||||
.where('userId', user.id)
|
||||
.first()
|
||||
|
||||
if (!keyword) {
|
||||
await ctx.reply(`Ключевое слово с ID ${keywordId} не найдено.`)
|
||||
return
|
||||
}
|
||||
|
||||
// Soft delete - set isActive to false
|
||||
keyword.isActive = false
|
||||
await keyword.save()
|
||||
|
||||
await ctx.reply(`Ключевое слово "${keyword.keyword}" (ID: ${keywordId}) успешно удалено.`)
|
||||
|
||||
logger.info(
|
||||
`Keyword deleted: "${keyword.keyword}" (ID: ${keywordId}) for user ${user.id} (chat_id: ${chatId})`
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle /help command - Show help message
|
||||
*/
|
||||
private async handleHelp(ctx: Context): Promise<void> {
|
||||
const helpMessage =
|
||||
'🤖 Помощь по использованию бота\n\n' +
|
||||
'Доступные команды:\n\n' +
|
||||
'/start - Регистрация в системе\n' +
|
||||
'/addkeyword <слово> - Добавить ключевое слово для отслеживания\n' +
|
||||
'/listkeywords - Список ваших ключевых слов\n' +
|
||||
'/deletekeyword <id> - Удалить ключевое слово\n' +
|
||||
'/help - Показать эту справку\n\n' +
|
||||
'Примеры использования:\n' +
|
||||
'/addkeyword строительство\n' +
|
||||
'/deletekeyword 5'
|
||||
|
||||
await ctx.reply(helpMessage)
|
||||
}
|
||||
|
||||
/**
|
||||
* Send notification to user about matched auction
|
||||
*/
|
||||
async sendNotification(
|
||||
chatId: string,
|
||||
auctionTitle: string,
|
||||
auctionUrl: string,
|
||||
keyword: string
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const message =
|
||||
`🔔 Новый аукцион по ключевому слову "${keyword}"\n\n` +
|
||||
`📋 ${auctionTitle}\n\n` +
|
||||
`🔗 ${auctionUrl}`
|
||||
|
||||
await this.bot.api.sendMessage(chatId, message, {
|
||||
parse_mode: 'HTML',
|
||||
})
|
||||
|
||||
logger.info(`Notification sent to chat ${chatId} for keyword "${keyword}"`)
|
||||
return true
|
||||
} catch (error) {
|
||||
logger.error('Failed to send notification', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
chatId,
|
||||
keyword,
|
||||
})
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the bot (long polling)
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (this.isRunning) {
|
||||
logger.warn('Bot is already running')
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info('Starting Telegram bot...')
|
||||
this.isRunning = true
|
||||
await this.bot.start()
|
||||
logger.info('Telegram bot started successfully')
|
||||
} catch (error) {
|
||||
this.isRunning = false
|
||||
logger.error('Failed to start Telegram bot', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the bot gracefully
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
if (!this.isRunning) {
|
||||
logger.warn('Bot is not running')
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info('Stopping Telegram bot...')
|
||||
await this.bot.stop()
|
||||
this.isRunning = false
|
||||
logger.info('Telegram bot stopped successfully')
|
||||
} catch (error) {
|
||||
logger.error('Failed to stop Telegram bot', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if bot is running
|
||||
*/
|
||||
getIsRunning(): boolean {
|
||||
return this.isRunning
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user