This commit is contained in:
Vakula Uladimir
2025-10-17 11:27:52 +03:00
commit 12f005e335
72 changed files with 14402 additions and 0 deletions
+324
View File
@@ -0,0 +1,324 @@
import type { HttpContext } from '@adonisjs/core/http'
import Auction from '#models/auction'
import ParseLog from '#models/parse_log'
import { errors as lucidErrors } from '@adonisjs/lucid'
import { DateTime } from 'luxon'
import { ScraperService } from '#services/scraper_service'
import { NotificationService } from '#services/notification_service'
import logger from '@adonisjs/core/services/logger'
export default class AuctionsController {
/**
* List auctions with pagination
* GET /api/auctions?page=1&limit=20
*/
async index({ request, response }: HttpContext) {
const page = request.input('page', 1)
const limit = request.input('limit', 20)
const pageNumber = Math.max(1, Number(page))
const limitNumber = Math.min(100, Math.max(1, Number(limit)))
try {
const auctions = await Auction.query()
.orderBy('created_at', 'desc')
.paginate(pageNumber, limitNumber)
return response.ok(auctions.serialize())
} catch (error) {
return response.internalServerError({
error: 'Failed to fetch auctions',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
/**
* Get a single auction by id with relationships
* GET /api/auctions/:id
*/
async show({ params, response }: HttpContext) {
const auctionId = params.id
if (!auctionId || isNaN(Number(auctionId))) {
return response.badRequest({
error: 'Invalid auction id',
})
}
try {
const auction = await Auction.query()
.where('id', auctionId)
.preload('notifications')
.firstOrFail()
return response.ok(auction.serialize())
} catch (error) {
if (error instanceof lucidErrors.E_ROW_NOT_FOUND) {
return response.notFound({
error: 'Auction not found',
})
}
return response.internalServerError({
error: 'Failed to fetch auction',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
/**
* Search auctions by keyword/title/status
* GET /api/auctions/search?q=keyword&status=active&page=1&limit=20
*/
async search({ request, response }: HttpContext) {
const searchQuery = request.input('q', '')
const status = request.input('status')
const page = request.input('page', 1)
const limit = request.input('limit', 20)
const pageNumber = Math.max(1, Number(page))
const limitNumber = Math.min(100, Math.max(1, Number(limit)))
try {
const query = Auction.query()
if (searchQuery && typeof searchQuery === 'string' && searchQuery.trim().length > 0) {
const trimmedQuery = searchQuery.trim()
query.where((builder) => {
builder
.whereILike('title', `%${trimmedQuery}%`)
.orWhereILike('description', `%${trimmedQuery}%`)
.orWhereILike('auction_num', `%${trimmedQuery}%`)
.orWhereILike('organization', `%${trimmedQuery}%`)
})
}
if (status && typeof status === 'string' && status.trim().length > 0) {
query.where('status', status.trim())
}
const auctions = await query.orderBy('created_at', 'desc').paginate(pageNumber, limitNumber)
return response.ok(auctions.serialize())
} catch (error) {
return response.internalServerError({
error: 'Failed to search auctions',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
/**
* List all auctions from last 3 days
* GET /list
*/
async list({ response }: HttpContext) {
try {
const threeDaysAgo = DateTime.now().minus({ days: 3 })
const auctions = await Auction.query()
.where('created_at', '>=', threeDaysAgo.toSQL())
.orderBy('created_at', 'desc')
return response.ok({
data: auctions.map((auction) => auction.serialize()),
meta: {
total: auctions.length,
from_date: threeDaysAgo.toISO(),
},
})
} catch (error) {
return response.internalServerError({
error: 'Failed to fetch recent auctions',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
/**
* Render HTML view of auctions from last 3 days
* GET /list-view
*/
async listView({ view }: HttpContext) {
const threeDaysAgo = DateTime.now().minus({ days: 3 })
const auctions = await Auction.query()
.where('created_at', '>=', threeDaysAgo.toSQL())
.orderBy('created_at', 'desc')
return view.render('auctions/list', {
auctions,
fromDate: threeDaysAgo.toFormat('dd.MM.yyyy HH:mm'),
})
}
/**
* Trigger auction parsing from web interface
* POST /trigger-parse
*/
async triggerParse({ request, response, session }: HttpContext) {
try {
// Validate input
const pages = request.input('pages', 1)
const notifySubscribers = request.input('notifySubscribers') === 'on'
const pagesNumber = Math.max(1, Math.min(10, Number(pages)))
if (isNaN(pagesNumber)) {
session.flash('error', 'Invalid number of pages')
return response.redirect('/list-view')
}
logger.info(`Parsing triggered from web interface`, {
pages: pagesNumber,
notifySubscribers,
})
const startTime = DateTime.now()
const scraper = new ScraperService()
const notificationService = new NotificationService()
// Create ParseLog entry
const parseLog = await ParseLog.create({
parseType: 'auction',
status: 'running',
itemsFound: 0,
errors: null,
startedAt: startTime,
completedAt: null,
})
let newAuctions = 0
let updatedAuctions = 0
let totalScraped = 0
const errors: string[] = []
// Step 1: Scrape auctions
logger.info(`Scraping ${pagesNumber} page(s) from icetrade.by`)
const auctionData = await scraper.scrapeAuctions(pagesNumber)
totalScraped = auctionData.length
if (totalScraped === 0) {
await this.updateParseLog(parseLog, 'completed', 0, null)
session.flash('error', 'No auctions found during parsing')
return response.redirect('/list-view')
}
// Step 2: Upsert auctions to database
logger.info(`Saving ${totalScraped} auction(s) to database`)
for (const data of auctionData) {
try {
// Check if auction exists by auctionNum
const existingAuction = await Auction.query()
.where('auctionNum', data.auctionNum)
.first()
let auction: Auction
if (existingAuction) {
// Update existing auction
existingAuction.title = data.title
existingAuction.description = data.description
existingAuction.organization = data.organization
existingAuction.status = data.status
existingAuction.deadline = data.deadline ? DateTime.fromISO(data.deadline) : null
existingAuction.url = data.link
existingAuction.rawData = data
await existingAuction.save()
auction = existingAuction
updatedAuctions++
logger.debug(`Updated auction: ${data.auctionNum}`)
} else {
// Create new auction
auction = await Auction.create({
auctionNum: data.auctionNum,
title: data.title,
description: data.description,
organization: data.organization,
status: data.status,
price: null,
deadline: data.deadline ? DateTime.fromISO(data.deadline) : null,
url: data.link,
rawData: data,
})
newAuctions++
logger.debug(`Created auction: ${data.auctionNum}`)
// Step 3: Check for keyword matches and send notifications (for new auctions only)
if (notifySubscribers) {
try {
await notificationService.checkAndNotify(auction)
} catch (error) {
const errorMsg = `Notification check failed for auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
logger.error(errorMsg)
errors.push(errorMsg)
// Don't fail the entire process if notification fails
}
}
}
} catch (error) {
const errorMsg = `Failed to save auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
logger.error(errorMsg)
errors.push(errorMsg)
}
}
// Update ParseLog with success status
const status = errors.length > 0 ? 'completed_with_errors' : 'completed'
await this.updateParseLog(
parseLog,
status,
newAuctions + updatedAuctions,
errors.length > 0 ? errors.join('\n') : null
)
// Create success message
let successMessage = `Parsing completed: ${totalScraped} auctions scraped, ${newAuctions} new, ${updatedAuctions} updated`
if (!notifySubscribers) {
successMessage += ' (notifications disabled)'
}
if (errors.length > 0) {
successMessage += ` with ${errors.length} error(s)`
}
session.flash('success', successMessage)
logger.info(successMessage)
return response.redirect('/list-view')
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error during parsing'
logger.error('Error in triggerParse', {
error: errorMessage,
})
session.flash('error', `Parsing failed: ${errorMessage}`)
return response.redirect('/list-view')
}
}
/**
* Update ParseLog entry with final status
*/
private async updateParseLog(
parseLog: ParseLog,
status: string,
itemsFound: number,
errors: string | null
): Promise<void> {
parseLog.status = status
parseLog.itemsFound = itemsFound
parseLog.errors = errors
parseLog.completedAt = DateTime.now()
await parseLog.save()
logger.info(`Updated ParseLog ${parseLog.id}: status=${status}, items=${itemsFound}`)
}
}
+101
View File
@@ -0,0 +1,101 @@
import type { HttpContext } from '@adonisjs/core/http'
import Keyword from '#models/keyword'
export default class KeywordsController {
/**
* List keywords by user_id with optional filtering by is_active
* GET /api/keywords?user_id=1&is_active=true
*/
async index({ request, response }: HttpContext) {
const userId = request.input('user_id')
const isActive = request.input('is_active')
if (!userId) {
return response.badRequest({
error: 'user_id query parameter is required',
})
}
const query = Keyword.query().where('user_id', userId)
if (isActive !== undefined) {
const activeValue = isActive === 'true' || isActive === true
query.where('is_active', activeValue)
}
const keywords = await query.orderBy('created_at', 'desc')
return response.ok(keywords)
}
/**
* Create a new keyword
* POST /api/keywords
* Body: { user_id: number, keyword: string, case_sensitive?: boolean }
*/
async store({ request, response }: HttpContext) {
const userId = request.input('user_id')
const keyword = request.input('keyword')
const caseSensitive = request.input('case_sensitive', false)
if (!userId || !keyword) {
return response.badRequest({
error: 'user_id and keyword are required',
})
}
if (typeof keyword !== 'string' || keyword.trim().length === 0) {
return response.badRequest({
error: 'keyword must be a non-empty string',
})
}
try {
const newKeyword = await Keyword.create({
userId,
keyword: keyword.trim(),
caseSensitive,
isActive: true,
})
return response.created(newKeyword)
} catch (error) {
return response.internalServerError({
error: 'Failed to create keyword',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
/**
* Delete a keyword by id
* DELETE /api/keywords/:id
*/
async destroy({ params, response }: HttpContext) {
const keywordId = params.id
if (!keywordId || isNaN(Number(keywordId))) {
return response.badRequest({
error: 'Invalid keyword id',
})
}
try {
const keyword = await Keyword.findOrFail(keywordId)
await keyword.delete()
return response.noContent()
} catch (error) {
if (error.code === 'E_ROW_NOT_FOUND') {
return response.notFound({
error: 'Keyword not found',
})
}
return response.internalServerError({
error: 'Failed to delete keyword',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
}
+29
View File
@@ -0,0 +1,29 @@
import type { HttpContext } from '@adonisjs/core/http'
import ParseLog from '#models/parse_log'
export default class ParseLogsController {
/**
* List parse logs with pagination, ordered by latest
* GET /api/parse-logs?page=1&limit=20
*/
async index({ request, response }: HttpContext) {
const page = request.input('page', 1)
const limit = request.input('limit', 20)
const pageNumber = Math.max(1, Number(page))
const limitNumber = Math.min(100, Math.max(1, Number(limit)))
try {
const parseLogs = await ParseLog.query()
.orderBy('started_at', 'desc')
.paginate(pageNumber, limitNumber)
return response.ok(parseLogs.serialize())
} catch (error) {
return response.internalServerError({
error: 'Failed to fetch parse logs',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
}
+49
View File
@@ -0,0 +1,49 @@
import app from '@adonisjs/core/services/app'
import { HttpContext, ExceptionHandler } from '@adonisjs/core/http'
import type { StatusPageRange, StatusPageRenderer } from '@adonisjs/core/types/http'
export default class HttpExceptionHandler extends ExceptionHandler {
/**
* In debug mode, the exception handler will display verbose errors
* with pretty printed stack traces.
*/
protected debug = !app.inProduction
/**
* Status pages are used to display a custom HTML pages for certain error
* codes. You might want to enable them in production only, but feel
* free to enable them in development as well.
*/
protected renderStatusPages = app.inProduction
/**
* Status pages is a collection of error code range and a callback
* to return the HTML contents to send as a response.
*/
protected statusPages: Record<StatusPageRange, StatusPageRenderer> = {
'404': (error, { view }) => {
return view.render('pages/errors/not_found', { error })
},
'500..599': (error, { view }) => {
return view.render('pages/errors/server_error', { error })
},
}
/**
* The method is used for handling errors and returning
* response to the client
*/
async handle(error: unknown, ctx: HttpContext) {
return super.handle(error, ctx)
}
/**
* The method is used to report error to the logging service or
* the a third party error monitoring service.
*
* @note You should not attempt to send a response from this method.
*/
async report(error: unknown, ctx: HttpContext) {
return super.report(error, ctx)
}
}
+25
View File
@@ -0,0 +1,25 @@
import type { HttpContext } from '@adonisjs/core/http'
import type { NextFn } from '@adonisjs/core/types/http'
import type { Authenticators } from '@adonisjs/auth/types'
/**
* Auth middleware is used authenticate HTTP requests and deny
* access to unauthenticated users.
*/
export default class AuthMiddleware {
/**
* The URL to redirect to, when authentication fails
*/
redirectTo = '/login'
async handle(
ctx: HttpContext,
next: NextFn,
options: {
guards?: (keyof Authenticators)[]
} = {}
) {
await ctx.auth.authenticateUsing(options.guards, { loginRoute: this.redirectTo })
return next()
}
}
@@ -0,0 +1,19 @@
import { Logger } from '@adonisjs/core/logger'
import { HttpContext } from '@adonisjs/core/http'
import { NextFn } from '@adonisjs/core/types/http'
/**
* The container bindings middleware binds classes to their request
* specific value using the container resolver.
*
* - We bind "HttpContext" class to the "ctx" object
* - And bind "Logger" class to the "ctx.logger" object
*/
export default class ContainerBindingsMiddleware {
handle(ctx: HttpContext, next: NextFn) {
ctx.containerResolver.bindValue(HttpContext, ctx)
ctx.containerResolver.bindValue(Logger, ctx.logger)
return next()
}
}
+31
View File
@@ -0,0 +1,31 @@
import type { HttpContext } from '@adonisjs/core/http'
import type { NextFn } from '@adonisjs/core/types/http'
import type { Authenticators } from '@adonisjs/auth/types'
/**
* Guest middleware is used to deny access to routes that should
* be accessed by unauthenticated users.
*
* For example, the login page should not be accessible if the user
* is already logged-in
*/
export default class GuestMiddleware {
/**
* The URL to redirect to when user is logged-in
*/
redirectTo = '/'
async handle(
ctx: HttpContext,
next: NextFn,
options: { guards?: (keyof Authenticators)[] } = {}
) {
for (let guard of options.guards || [ctx.auth.defaultGuard]) {
if (await ctx.auth.use(guard).check()) {
return ctx.response.redirect(this.redirectTo, true)
}
}
return next()
}
}
+19
View File
@@ -0,0 +1,19 @@
import type { HttpContext } from '@adonisjs/core/http'
import type { NextFn } from '@adonisjs/core/types/http'
/**
* Silent auth middleware can be used as a global middleware to silent check
* if the user is logged-in or not.
*
* The request continues as usual, even when the user is not logged-in.
*/
export default class SilentAuthMiddleware {
async handle(
ctx: HttpContext,
next: NextFn,
) {
await ctx.auth.check()
return next()
}
}
+48
View File
@@ -0,0 +1,48 @@
import { DateTime } from 'luxon'
import { BaseModel, column, hasMany } from '@adonisjs/lucid/orm'
import type { HasMany } from '@adonisjs/lucid/types/relations'
import Notification from '#models/notification'
export default class Auction extends BaseModel {
@column({ isPrimary: true })
declare id: number
@column()
declare auctionNum: string
@column()
declare title: string
@column()
declare description: string | null
@column()
declare organization: string | null
@column()
declare status: string | null
@column()
declare price: number | null
@column.dateTime()
declare deadline: DateTime | null
@column()
declare url: string | null
@column({
prepare: (value: Record<string, any> | null) => (value ? JSON.stringify(value) : null),
consume: (value: string | null) => (value ? JSON.parse(value) : null),
})
declare rawData: Record<string, any> | null
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime
@hasMany(() => Notification)
declare notifications: HasMany<typeof Notification>
}
+30
View File
@@ -0,0 +1,30 @@
import { DateTime } from 'luxon'
import { BaseModel, column, belongsTo } from '@adonisjs/lucid/orm'
import type { BelongsTo } from '@adonisjs/lucid/types/relations'
import User from '#models/user'
export default class Keyword extends BaseModel {
@column({ isPrimary: true })
declare id: number
@column()
declare userId: number
@column()
declare keyword: string
@column()
declare isActive: boolean
@column()
declare caseSensitive: boolean
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime
@belongsTo(() => User)
declare user: BelongsTo<typeof User>
}
+37
View File
@@ -0,0 +1,37 @@
import { DateTime } from 'luxon'
import { BaseModel, column, belongsTo } from '@adonisjs/lucid/orm'
import type { BelongsTo } from '@adonisjs/lucid/types/relations'
import Auction from '#models/auction'
import Keyword from '#models/keyword'
export default class Notification extends BaseModel {
@column({ isPrimary: true })
declare id: number
@column()
declare auctionId: number
@column()
declare keywordId: number
@column()
declare status: 'pending' | 'sent' | 'failed'
@column()
declare errorMessage: string | null
@column.dateTime()
declare sentAt: DateTime | null
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime
@belongsTo(() => Auction)
declare auction: BelongsTo<typeof Auction>
@belongsTo(() => Keyword)
declare keyword: BelongsTo<typeof Keyword>
}
+31
View File
@@ -0,0 +1,31 @@
import { DateTime } from 'luxon'
import { BaseModel, column } from '@adonisjs/lucid/orm'
export default class ParseLog extends BaseModel {
@column({ isPrimary: true })
declare id: number
@column()
declare parseType: string
@column()
declare status: string
@column()
declare itemsFound: number
@column()
declare errors: string | null
@column.dateTime()
declare startedAt: DateTime
@column.dateTime()
declare completedAt: DateTime | null
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime
}
+30
View File
@@ -0,0 +1,30 @@
import { DateTime } from 'luxon'
import hash from '@adonisjs/core/services/hash'
import { compose } from '@adonisjs/core/helpers'
import { BaseModel, column } from '@adonisjs/lucid/orm'
import { withAuthFinder } from '@adonisjs/auth/mixins/lucid'
const AuthFinder = withAuthFinder(() => hash.use('scrypt'), {
uids: ['email'],
passwordColumnName: 'password',
})
export default class User extends compose(BaseModel, AuthFinder) {
@column({ isPrimary: true })
declare id: number
@column()
declare email: string
@column({ serializeAs: null })
declare password: string
@column()
declare telegramChatId: string | null
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime
}
+20
View File
@@ -0,0 +1,20 @@
import { z } from 'zod'
/**
* Zod schema for auction data validation
* Ensures all scraped data conforms to expected structure before database insertion
*/
export const AuctionSchema = z.object({
auctionNum: z.string().trim().min(1, 'Auction number is required'),
title: z.string().trim().min(1, 'Title is required'),
organization: z.string().trim().min(1, 'Organization is required'),
status: z.string().trim().min(1, 'Status is required'),
deadline: z.string().nullable().default(null),
link: z.string().url('Link must be a valid URL'),
description: z.string().nullable().default(null),
})
/**
* TypeScript type inferred from schema
*/
export type AuctionData = z.infer<typeof AuctionSchema>
+223
View File
@@ -0,0 +1,223 @@
import logger from '@adonisjs/core/services/logger'
import { DateTime } from 'luxon'
import Auction from '#models/auction'
import Keyword from '#models/keyword'
import Notification from '#models/notification'
import User from '#models/user'
import { TelegramService } from '#services/telegram_service'
/**
* NotificationService - Handles keyword matching and notification delivery
*
* Features:
* - Matches keywords against auction titles and descriptions
* - Case-sensitive and case-insensitive matching support
* - Prevents duplicate notifications (checks existing records)
* - Tracks notification status (pending/sent/failed)
* - Integrates with TelegramService for message delivery
* - Validates user active status before sending
*/
export class NotificationService {
/**
* Check auction against all active keywords and send notifications for matches
*
* @param auction - The auction to check for keyword matches
*/
async checkAndNotify(auction: Auction): Promise<void> {
try {
logger.info(`Checking auction ${auction.auctionNum} for keyword matches`)
// Get all active keywords
const keywords = await Keyword.query().where('isActive', true).preload('user')
if (keywords.length === 0) {
logger.debug('No active keywords found, skipping notification check')
return
}
logger.debug(`Found ${keywords.length} active keyword(s) to check`)
let matchCount = 0
for (const keyword of keywords) {
try {
// Check if keyword matches auction title or description
const titleMatch = this.matchKeyword(
auction.title,
keyword.keyword,
keyword.caseSensitive
)
const descMatch = auction.description
? this.matchKeyword(auction.description, keyword.keyword, keyword.caseSensitive)
: false
if (!titleMatch && !descMatch) {
continue // No match, skip this keyword
}
matchCount++
logger.info(
`Keyword match found: "${keyword.keyword}" in auction ${auction.auctionNum}`,
{
keywordId: keyword.id,
auctionId: auction.id,
matchedIn: titleMatch ? 'title' : 'description',
}
)
// Check if notification already exists (prevent duplicates)
const existingNotification = await Notification.query()
.where('auctionId', auction.id)
.where('keywordId', keyword.id)
.first()
if (existingNotification) {
logger.debug(
`Notification already exists for auction ${auction.id} and keyword ${keyword.id}, skipping`,
{
notificationId: existingNotification.id,
status: existingNotification.status,
}
)
continue
}
// Create notification record with 'pending' status
const notification = await Notification.create({
auctionId: auction.id,
keywordId: keyword.id,
status: 'pending',
errorMessage: null,
sentAt: null,
})
logger.debug(`Created notification record ${notification.id}`)
// Get user and validate active status
const user = await User.find(keyword.userId)
if (!user) {
logger.warn(`User ${keyword.userId} not found for keyword ${keyword.id}`)
await notification
.merge({
status: 'failed',
errorMessage: 'User not found',
})
.save()
continue
}
if (!user.telegramChatId) {
logger.warn(`User ${user.id} has no Telegram chat ID`)
await notification
.merge({
status: 'failed',
errorMessage: 'User has no Telegram chat ID',
})
.save()
continue
}
// Send notification via Telegram
try {
const telegramService = TelegramService.getInstance()
const auctionUrl = auction.url || `https://icetrade.by/auction/${auction.auctionNum}`
const success = await telegramService.sendNotification(
user.telegramChatId,
auction.title,
auctionUrl,
keyword.keyword
)
if (success) {
// Update notification status to 'sent'
await notification
.merge({
status: 'sent',
sentAt: DateTime.now(),
})
.save()
logger.info(`Notification sent successfully`, {
notificationId: notification.id,
userId: user.id,
chatId: user.telegramChatId,
})
} else {
// Update notification status to 'failed'
await notification
.merge({
status: 'failed',
errorMessage: 'Failed to send Telegram message (unknown error)',
})
.save()
logger.error(`Failed to send notification`, {
notificationId: notification.id,
userId: user.id,
chatId: user.telegramChatId,
})
}
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : 'Unknown error during notification send'
logger.error(`Error sending notification`, {
notificationId: notification.id,
error: errorMessage,
userId: user.id,
})
// Update notification status to 'failed' with error details
await notification
.merge({
status: 'failed',
errorMessage,
})
.save()
}
} catch (error) {
logger.error(`Error processing keyword ${keyword.id} for auction ${auction.id}`, {
error: error instanceof Error ? error.message : String(error),
keywordId: keyword.id,
auctionId: auction.id,
})
// Continue processing other keywords even if one fails
continue
}
}
logger.info(
`Completed notification check for auction ${auction.auctionNum}: ${matchCount} match(es) found`
)
} catch (error) {
logger.error(`Error in checkAndNotify for auction ${auction.id}`, {
error: error instanceof Error ? error.message : String(error),
auctionId: auction.id,
})
// Don't throw - allow scraper to continue processing other auctions
}
}
/**
* Check if a keyword matches text with case-sensitive or case-insensitive matching
*
* @param text - The text to search in
* @param keyword - The keyword to search for
* @param caseSensitive - Whether to perform case-sensitive matching
* @returns true if keyword is found in text, false otherwise
*/
private matchKeyword(text: string, keyword: string, caseSensitive: boolean): boolean {
if (!text || !keyword) {
return false
}
if (caseSensitive) {
return text.includes(keyword)
}
return text.toLowerCase().includes(keyword.toLowerCase())
}
}
+338
View File
@@ -0,0 +1,338 @@
import * as cheerio from 'cheerio'
import logger from '@adonisjs/core/services/logger'
import { AuctionSchema, type AuctionData } from '../schemas/auction_schema.js'
/**
* Configuration for the scraper
*/
const SCRAPER_CONFIG = {
baseUrl: 'https://icetrade.by',
requestDelay: 1000, // 1 second between requests
timeout: 30000, // 30 seconds timeout
userAgent:
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
maxRetries: 3,
retryDelay: 2000,
} as const
/**
* Error thrown when scraping fails
*/
export class ScraperError extends Error {
constructor(
message: string,
public readonly cause?: unknown
) {
super(message)
this.name = 'ScraperError'
}
}
/**
* ScraperService - Handles fetching and parsing auction data from icetrade.by
*
* Features:
* - Rate limiting (1s delay between requests)
* - Retry logic with exponential backoff
* - Comprehensive error handling
* - Data validation with Zod
* - Structured logging
*/
export class ScraperService {
/**
* Builds the URL for fetching auctions with all required parameters
*/
private buildUrl(pageNumber: number): string {
const params = new URLSearchParams({
search_text: '',
'zakup_type[1]': '1',
'zakup_type[2]': '1',
onPage: '100',
sort: 'num:desc',
p: pageNumber.toString(),
// Status flags
'r[1]': '1',
'r[2]': '1',
'r[3]': '1',
'r[4]': '1',
'r[5]': '1',
'r[6]': '1',
'r[7]': '1',
// Trade type flags
't[Trade]': '1',
't[contest]': '1',
't[request]': '1',
't[qualification]': '1',
't[negotiations]': '1',
})
return `${SCRAPER_CONFIG.baseUrl}/trades/index?${params.toString()}`
}
/**
* Fetches HTML content from the specified page with retry logic
*/
async fetchPage(pageNumber: number): Promise<string> {
const url = this.buildUrl(pageNumber)
let lastError: Error | undefined
logger.info(`Preparing to fetch URL: ${url}`)
for (let attempt = 1; attempt <= SCRAPER_CONFIG.maxRetries; attempt++) {
try {
logger.info(
`Fetching page ${pageNumber} (attempt ${attempt}/${SCRAPER_CONFIG.maxRetries})`,
{ url }
)
const controller = new AbortController()
const timeoutId = setTimeout(() => controller.abort(), SCRAPER_CONFIG.timeout)
const response = await fetch(url, {
headers: {
'User-Agent': SCRAPER_CONFIG.userAgent,
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
Connection: 'keep-alive',
},
signal: controller.signal,
})
clearTimeout(timeoutId)
logger.info(`Received response: status=${response.status} ${response.statusText}`)
if (!response.ok) {
const bodyText = await response.text().catch(() => 'Unable to read response body')
throw new Error(
`HTTP ${response.status}: ${response.statusText}. Body: ${bodyText.substring(0, 200)}`
)
}
const html = await response.text()
if (!html || html.trim().length === 0) {
throw new Error('Received empty response')
}
logger.info(`Successfully fetched page ${pageNumber}: ${html.length} bytes`)
return html
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error))
// Log detailed error information
const errorDetails: Record<string, any> = {
message: lastError.message,
name: lastError.name,
url,
}
// Add stack trace for non-HTTP errors
if (!(lastError.message.startsWith('HTTP '))) {
errorDetails.stack = lastError.stack
}
// Check for specific error types
if (lastError.name === 'AbortError') {
errorDetails.reason = 'Request timeout after 30s'
} else if (lastError.message.includes('fetch failed')) {
errorDetails.reason = 'Network error - check DNS, firewall, or connectivity'
} else if (lastError.message.includes('ENOTFOUND')) {
errorDetails.reason = 'DNS resolution failed - domain not found'
} else if (lastError.message.includes('ECONNREFUSED')) {
errorDetails.reason = 'Connection refused - server not reachable'
} else if (lastError.message.includes('ETIMEDOUT')) {
errorDetails.reason = 'Connection timeout - server too slow or unreachable'
}
logger.warn(errorDetails, `Failed to fetch page ${pageNumber} (attempt ${attempt}/${SCRAPER_CONFIG.maxRetries})`)
if (attempt < SCRAPER_CONFIG.maxRetries) {
const delay = SCRAPER_CONFIG.retryDelay * attempt
logger.info(`Retrying in ${delay}ms...`)
await this.delay(delay)
}
}
}
const finalError = new ScraperError(
`Failed to fetch page ${pageNumber} after ${SCRAPER_CONFIG.maxRetries} attempts: ${lastError?.message}`,
lastError
)
logger.error('All fetch attempts failed', {
pageNumber,
url,
lastErrorMessage: lastError?.message,
lastErrorName: lastError?.name,
})
throw finalError
}
/**
* Parses HTML content and extracts auction data
*/
parsePage(html: string): AuctionData[] {
try {
const $ = cheerio.load(html)
const auctions: AuctionData[] = []
// Find the auctions table
const auctionsTable = $('table.auctions.w100')
if (auctionsTable.length === 0) {
logger.warn('No auctions table found in HTML')
return []
}
// Parse each auction row
const rows = auctionsTable.find('tbody tr')
logger.info(`Found ${rows.length} auction rows to parse`)
rows.each((index, element) => {
try {
const row = $(element)
// Extract auction data from table cells
const cells = row.find('td')
if (cells.length < 4) {
logger.warn(`Row ${index} has insufficient cells, skipping`)
return
}
// Extract auction number (typically in first cell)
const auctionNumCell = $(cells[0])
const auctionNum = auctionNumCell.text().trim()
// Extract title and link (typically in second cell with <a> tag)
const titleCell = $(cells[1])
const titleLink = titleCell.find('a').first()
const title = titleLink.text().trim()
const link = titleLink.attr('href')?.trim() || ''
// Skip if link is empty or missing
if (!link) {
logger.warn(`Row ${index} has missing or empty link, skipping`)
return
}
// Make link absolute if it's relative
const absoluteLink = link.startsWith('http')
? link
: `${SCRAPER_CONFIG.baseUrl}${link.startsWith('/') ? link : `/${link}`}`
// Extract organization (typically in third cell)
const organizationCell = $(cells[2])
const organization = organizationCell.text().trim()
// Extract status (typically in fourth cell)
const statusCell = $(cells[3])
const status = statusCell.text().trim()
// Extract deadline if available (typically in fifth cell)
const deadlineCell = $(cells[4])
const deadline = deadlineCell.text().trim() || null
// Description can be extracted from title cell's additional text or separate element
const description = titleCell.find('.description').text().trim() || null
// Validate with Zod schema
const result = AuctionSchema.safeParse({
auctionNum,
title,
organization,
status,
deadline,
link: absoluteLink,
description,
})
if (!result.success) {
logger.warn(`Validation failed for auction at row ${index}`, {
errors: result.error.issues,
data: { auctionNum, title },
})
return
}
auctions.push(result.data)
} catch (error) {
logger.error(`Error parsing auction row ${index}`, {
error: error instanceof Error ? error.message : String(error),
})
}
})
logger.info(`Successfully parsed ${auctions.length} valid auctions`)
return auctions
} catch (error) {
throw new ScraperError(
'Failed to parse HTML content',
error instanceof Error ? error : new Error(String(error))
)
}
}
/**
* Scrapes multiple pages of auctions with rate limiting
*
* @param maxPages - Maximum number of pages to scrape (default: 1)
* @returns Array of all parsed auction data
*/
async scrapeAuctions(maxPages: number = 1): Promise<AuctionData[]> {
if (maxPages < 1) {
throw new Error('maxPages must be at least 1')
}
logger.info(`Starting scrape of ${maxPages} page(s)`)
const allAuctions: AuctionData[] = []
for (let page = 1; page <= maxPages; page++) {
try {
// Fetch page HTML
const html = await this.fetchPage(page)
// Parse auctions from HTML
const auctions = this.parsePage(html)
allAuctions.push(...auctions)
logger.info(`Page ${page}/${maxPages}: Found ${auctions.length} auctions`)
// Rate limiting: wait before next request (except for last page)
if (page < maxPages) {
logger.debug(`Waiting ${SCRAPER_CONFIG.requestDelay}ms before next request`)
await this.delay(SCRAPER_CONFIG.requestDelay)
}
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error))
logger.error(`Error scraping page ${page}`, {
message: err.message,
name: err.name,
stack: err.stack,
cause: err.cause,
})
// Continue to next page instead of failing completely
// This ensures partial data is still returned
continue
}
}
logger.info(
`Scraping completed: ${allAuctions.length} total auctions from ${maxPages} page(s)`
)
return allAuctions
}
/**
* Helper method for delays (rate limiting, retries)
*/
private delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms))
}
}
+404
View File
@@ -0,0 +1,404 @@
import { Bot, Context } from 'grammy'
import env from '#start/env'
import logger from '@adonisjs/core/services/logger'
import User from '#models/user'
import Keyword from '#models/keyword'
/**
* TelegramService - Handles Telegram bot operations
*
* Singleton service for managing Grammy bot instance and commands.
* Provides keyword management and notification delivery.
*/
export class TelegramService {
private static instance: TelegramService | null = null
private bot: Bot
private isRunning: boolean = false
private constructor() {
const token = env.get('TELEGRAM_BOT_TOKEN')
this.bot = new Bot(token)
this.setupCommands()
}
/**
* Get singleton instance of TelegramService
*/
static getInstance(): TelegramService {
if (!TelegramService.instance) {
TelegramService.instance = new TelegramService()
}
return TelegramService.instance
}
/**
* Setup bot commands and handlers
*/
private setupCommands(): void {
// /start - Register user
this.bot.command('start', async (ctx) => {
try {
await this.handleStart(ctx)
} catch (error) {
logger.error('Error handling /start command', {
error: error instanceof Error ? error.message : String(error),
chatId: ctx.chat?.id,
})
await ctx.reply('Произошла ошибка при регистрации. Попробуйте позже.')
}
})
// /addkeyword - Add keyword
this.bot.command('addkeyword', async (ctx) => {
try {
await this.handleAddKeyword(ctx)
} catch (error) {
logger.error('Error handling /addkeyword command', {
error: error instanceof Error ? error.message : String(error),
chatId: ctx.chat?.id,
})
await ctx.reply('Произошла ошибка при добавлении ключевого слова. Попробуйте позже.')
}
})
// /listkeywords - List keywords
this.bot.command('listkeywords', async (ctx) => {
try {
await this.handleListKeywords(ctx)
} catch (error) {
logger.error('Error handling /listkeywords command', {
error: error instanceof Error ? error.message : String(error),
chatId: ctx.chat?.id,
})
await ctx.reply('Произошла ошибка при получении списка ключевых слов. Попробуйте позже.')
}
})
// /deletekeyword - Delete keyword
this.bot.command('deletekeyword', async (ctx) => {
try {
await this.handleDeleteKeyword(ctx)
} catch (error) {
logger.error('Error handling /deletekeyword command', {
error: error instanceof Error ? error.message : String(error),
chatId: ctx.chat?.id,
})
await ctx.reply('Произошла ошибка при удалении ключевого слова. Попробуйте позже.')
}
})
// /help - Show help
this.bot.command('help', async (ctx) => {
await this.handleHelp(ctx)
})
// Error handler
this.bot.catch((err) => {
logger.error('Grammy bot error', {
error: err.error instanceof Error ? err.error.message : String(err.error),
ctx: err.ctx,
})
})
}
/**
* Handle /start command - Register user
*/
private async handleStart(ctx: Context): Promise<void> {
if (!ctx.chat?.id) {
await ctx.reply('Не удалось определить chat ID.')
return
}
const chatId = String(ctx.chat.id)
// Check if user already exists
const existingUser = await User.findBy('telegramChatId', chatId)
if (existingUser) {
await ctx.reply(
'Вы уже зарегистрированы!\n\n' +
'Используйте /addkeyword для добавления ключевых слов.\n' +
'Используйте /help для получения справки.'
)
logger.info(`User already registered: ${chatId}`)
return
}
// Create new user
const user = await User.create({
email: `telegram_${chatId}@temp.local`,
password: Math.random().toString(36).substring(2, 15),
telegramChatId: chatId,
})
await ctx.reply(
'Добро пожаловать! Вы успешно зарегистрированы.\n\n' +
'Теперь вы можете добавлять ключевые слова для отслеживания аукционов:\n' +
'/addkeyword <слово> - добавить ключевое слово\n' +
'/listkeywords - список ваших ключевых слов\n' +
'/deletekeyword <id> - удалить ключевое слово\n' +
'/help - справка'
)
logger.info(`New user registered: ${chatId} (user_id: ${user.id})`)
}
/**
* Handle /addkeyword command - Add keyword for user
*/
private async handleAddKeyword(ctx: Context): Promise<void> {
if (!ctx.chat?.id) {
await ctx.reply('Не удалось определить chat ID.')
return
}
const chatId = String(ctx.chat.id)
const user = await User.findBy('telegramChatId', chatId)
if (!user) {
await ctx.reply('Вы не зарегистрированы. Используйте /start для регистрации.')
return
}
// Extract keyword from message
const messageText = ctx.message?.text || ''
const parts = messageText.split(' ')
if (parts.length < 2) {
await ctx.reply(
'Использование: /addkeyword <слово>\n\n' + 'Пример: /addkeyword строительство'
)
return
}
const keyword = parts.slice(1).join(' ').trim()
if (keyword.length === 0) {
await ctx.reply('Ключевое слово не может быть пустым.')
return
}
if (keyword.length > 255) {
await ctx.reply('Ключевое слово слишком длинное (максимум 255 символов).')
return
}
// Check if keyword already exists for this user
const existingKeyword = await Keyword.query()
.where('userId', user.id)
.where('keyword', keyword)
.first()
if (existingKeyword) {
await ctx.reply(`Ключевое слово "${keyword}" уже добавлено.`)
return
}
// Create keyword
const newKeyword = await Keyword.create({
userId: user.id,
keyword: keyword,
isActive: true,
})
await ctx.reply(`Ключевое слово "${keyword}" успешно добавлено (ID: ${newKeyword.id}).`)
logger.info(`Keyword added: "${keyword}" for user ${user.id} (chat_id: ${chatId})`)
}
/**
* Handle /listkeywords command - List user's keywords
*/
private async handleListKeywords(ctx: Context): Promise<void> {
if (!ctx.chat?.id) {
await ctx.reply('Не удалось определить chat ID.')
return
}
const chatId = String(ctx.chat.id)
const user = await User.findBy('telegramChatId', chatId)
if (!user) {
await ctx.reply('Вы не зарегистрированы. Используйте /start для регистрации.')
return
}
// Fetch user's keywords
const keywords = await Keyword.query().where('userId', user.id).where('isActive', true)
if (keywords.length === 0) {
await ctx.reply(
'У вас нет активных ключевых слов.\n\n' +
'Используйте /addkeyword для добавления ключевых слов.'
)
return
}
// Format keywords list
const keywordsList = keywords
.map((kw) => `${kw.id}. ${kw.keyword}`)
.join('\n')
await ctx.reply(`Ваши ключевые слова:\n\n${keywordsList}\n\nДля удаления используйте: /deletekeyword <id>`)
logger.info(`Listed ${keywords.length} keyword(s) for user ${user.id} (chat_id: ${chatId})`)
}
/**
* Handle /deletekeyword command - Delete keyword by ID
*/
private async handleDeleteKeyword(ctx: Context): Promise<void> {
if (!ctx.chat?.id) {
await ctx.reply('Не удалось определить chat ID.')
return
}
const chatId = String(ctx.chat.id)
const user = await User.findBy('telegramChatId', chatId)
if (!user) {
await ctx.reply('Вы не зарегистрированы. Используйте /start для регистрации.')
return
}
// Extract keyword ID from message
const messageText = ctx.message?.text || ''
const parts = messageText.split(' ')
if (parts.length < 2) {
await ctx.reply('Использование: /deletekeyword <id>\n\n' + 'Пример: /deletekeyword 5')
return
}
const keywordId = Number.parseInt(parts[1], 10)
if (Number.isNaN(keywordId)) {
await ctx.reply('Некорректный ID. Используйте числовое значение.')
return
}
// Find keyword
const keyword = await Keyword.query()
.where('id', keywordId)
.where('userId', user.id)
.first()
if (!keyword) {
await ctx.reply(`Ключевое слово с ID ${keywordId} не найдено.`)
return
}
// Soft delete - set isActive to false
keyword.isActive = false
await keyword.save()
await ctx.reply(`Ключевое слово "${keyword.keyword}" (ID: ${keywordId}) успешно удалено.`)
logger.info(
`Keyword deleted: "${keyword.keyword}" (ID: ${keywordId}) for user ${user.id} (chat_id: ${chatId})`
)
}
/**
* Handle /help command - Show help message
*/
private async handleHelp(ctx: Context): Promise<void> {
const helpMessage =
'🤖 Помощь по использованию бота\n\n' +
'Доступные команды:\n\n' +
'/start - Регистрация в системе\n' +
'/addkeyword <слово> - Добавить ключевое слово для отслеживания\n' +
'/listkeywords - Список ваших ключевых слов\n' +
'/deletekeyword <id> - Удалить ключевое слово\n' +
'/help - Показать эту справку\n\n' +
'Примеры использования:\n' +
'/addkeyword строительство\n' +
'/deletekeyword 5'
await ctx.reply(helpMessage)
}
/**
* Send notification to user about matched auction
*/
async sendNotification(
chatId: string,
auctionTitle: string,
auctionUrl: string,
keyword: string
): Promise<boolean> {
try {
const message =
`🔔 Новый аукцион по ключевому слову "${keyword}"\n\n` +
`📋 ${auctionTitle}\n\n` +
`🔗 ${auctionUrl}`
await this.bot.api.sendMessage(chatId, message, {
parse_mode: 'HTML',
})
logger.info(`Notification sent to chat ${chatId} for keyword "${keyword}"`)
return true
} catch (error) {
logger.error('Failed to send notification', {
error: error instanceof Error ? error.message : String(error),
chatId,
keyword,
})
return false
}
}
/**
* Start the bot (long polling)
*/
async start(): Promise<void> {
if (this.isRunning) {
logger.warn('Bot is already running')
return
}
try {
logger.info('Starting Telegram bot...')
this.isRunning = true
await this.bot.start()
logger.info('Telegram bot started successfully')
} catch (error) {
this.isRunning = false
logger.error('Failed to start Telegram bot', {
error: error instanceof Error ? error.message : String(error),
})
throw error
}
}
/**
* Stop the bot gracefully
*/
async stop(): Promise<void> {
if (!this.isRunning) {
logger.warn('Bot is not running')
return
}
try {
logger.info('Stopping Telegram bot...')
await this.bot.stop()
this.isRunning = false
logger.info('Telegram bot stopped successfully')
} catch (error) {
logger.error('Failed to stop Telegram bot', {
error: error instanceof Error ? error.message : String(error),
})
throw error
}
}
/**
* Check if bot is running
*/
getIsRunning(): boolean {
return this.isRunning
}
}