This commit is contained in:
Vakula Uladimir 2025-10-17 11:27:52 +03:00
commit 12f005e335
72 changed files with 14402 additions and 0 deletions

71
.dockerignore Normal file
View File

@ -0,0 +1,71 @@
# Node modules (will be installed in container)
node_modules
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Build output (will be generated in container)
build
dist
# Environment files (use docker-compose env instead)
.env
.env.*
!.env.example
# Development files
.vscode
.idea
*.swp
*.swo
*~
# Git
.git
.gitignore
.gitattributes
# Testing
coverage
.nyc_output
# OS files
.DS_Store
Thumbs.db
# Logs
logs
*.log
pids
*.pid
*.seed
*.pid.lock
# Documentation (not needed in runtime)
*.md
!DOCKER.md
# Docker files (avoid recursive copying)
Dockerfile
docker-compose.yml
.dockerignore
# Temporary files
tmp
temp
.cache
# IDE and editor files (keep build-critical configs)
.editorconfig
.prettierrc
# Note: tsconfig.json, .adonisrc.json are REQUIRED for build - do not exclude
# CI/CD
.github
.gitlab-ci.yml
.travis.yml
# Database files (use Docker volumes)
*.sqlite
*.sqlite3
*.db

22
.editorconfig Normal file
View File

@ -0,0 +1,22 @@
# http://editorconfig.org
[*]
indent_style = space
indent_size = 2
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.json]
insert_final_newline = unset
[**.min.js]
indent_style = unset
insert_final_newline = unset
[MakeFile]
indent_style = space
[*.md]
trim_trailing_whitespace = false

28
.env.docker Normal file
View File

@ -0,0 +1,28 @@
# Docker Environment Configuration
# Copy this file to .env before running docker-compose
# Application Settings
TZ=UTC
PORT=3333
LOG_LEVEL=info
NODE_ENV=production
SESSION_DRIVER=cookie
# Security - REQUIRED: Generate a secure APP_KEY
# Run: node ace generate:key
# Or generate a random 32-character string
APP_KEY=yECfcbOIfHA0nFEdYdj-1ixOIwC0jExoz
# Database Configuration
DB_HOST=postgres
DB_PORT=5432
DB_USER=postgres
DB_PASSWORD=postgres
DB_DATABASE=parser_zakupok
# Telegram Bot - REQUIRED
# Get token from @BotFather on Telegram
TELEGRAM_BOT_TOKEN=your_bot_token_here
# Target Website
ICETRADE_BASE_URL=https://icetrade.by

16
.env.example Normal file
View File

@ -0,0 +1,16 @@
TZ=UTC
PORT=3333
HOST=localhost
LOG_LEVEL=info
APP_KEY=
NODE_ENV=development
SESSION_DRIVER=cookie
DB_HOST=localhost
DB_PORT=5432
DB_USER=postgres
DB_PASSWORD=postgres
DB_DATABASE=parser_zakupok
TELEGRAM_BOT_TOKEN=your_bot_token_here
ICETRADE_BASE_URL=https://icetrade.by

27
.gitignore vendored Normal file
View File

@ -0,0 +1,27 @@
# Dependencies and AdonisJS build
node_modules
build
tmp
# Secrets
.env
.env.local
.env.production.local
.env.development.local
# Frontend assets compiled code
public/assets
# Build tools specific
npm-debug.log
yarn-error.log
# Editors specific
.fleet
.idea
.vscode
.claude
CLAUDE.md
# Platform specific
.DS_Store

65
Dockerfile Normal file
View File

@ -0,0 +1,65 @@
# Multi-stage build for AdonisJS 6 application
# Stage 1: Dependencies installation
FROM node:20-alpine AS dependencies
WORKDIR /app
# Copy package files
COPY package*.json ./
# Install all dependencies (including devDependencies for building)
RUN npm ci
# Stage 2: Build application
FROM node:20-alpine AS builder
WORKDIR /app
# Copy package files first
COPY package*.json ./
# Copy dependencies from previous stage
COPY --from=dependencies /app/node_modules ./node_modules
# Copy source code
COPY . .
# Build the application (compiles TypeScript and creates production build in build/)
# Using npm run build ensures proper environment setup
RUN npm run build
# Stage 3: Production runtime
FROM node:20-alpine AS production
# Install dumb-init for proper signal handling + ca-certificates for HTTPS
RUN apk add --no-cache dumb-init ca-certificates
WORKDIR /app
# Create non-root user for security
RUN addgroup -g 1001 -S nodejs && \
adduser -S adonisjs -u 1001
# Copy built application from builder stage
COPY --from=builder --chown=adonisjs:nodejs /app/build ./
# Install production dependencies only
# The build folder contains its own package.json with production dependencies
RUN npm ci --omit=dev
# Set environment to production
ENV NODE_ENV=production
ENV PORT=3333
ENV HOST=0.0.0.0
# Expose application port
EXPOSE 3333
# Switch to non-root user
USER adonisjs
# Use dumb-init to handle signals properly (for graceful shutdown)
ENTRYPOINT ["dumb-init", "--"]
# Start the application
CMD ["node", "bin/server.js"]

548
README.md Normal file
View File

@ -0,0 +1,548 @@
# Parser Zakupok (Парсер Закупок)
AdonisJS 6 приложение для автоматического парсинга аукционов с сайта icetrade.by. Система автоматически собирает данные каждые 6 часов, сохраняет в PostgreSQL и отправляет Telegram уведомления по ключевым словам.
## Возможности
- 🔄 Автоматический парсинг аукционов каждые 6 часов
- 🔍 Поиск по ключевым словам (заголовок, описание, организация)
- 📱 Telegram уведомления пользователям
- 📊 Логирование всех операций парсинга
- 🗄️ PostgreSQL база данных с полной историей
- ⚡ Пагинация и rate limiting для стабильности
- 🔧 REST API для управления
## Технологический стек
- **Framework**: AdonisJS 6 (TypeScript)
- **Database**: PostgreSQL + Lucid ORM
- **Scheduler**: adonisjs-scheduler
- **HTML Parser**: Cheerio
- **Validation**: Zod
- **Telegram Bot**: Grammy
- **Testing**: Japa
## Установка
### Требования
- Node.js >= 20.6
- PostgreSQL >= 14
- npm >= 9
### Шаги установки
```bash
# 1. Клонировать репозиторий
git clone <repository-url>
cd parser-zakupok
# 2. Установить зависимости
npm install
# 3. Скопировать .env файл
cp .env.example .env
# 4. Настроить переменные окружения
nano .env # или любой редактор
# 5. Запустить миграции
node ace migration:run
# 6. Запустить приложение
npm run dev
```
### Настройка .env
```env
# Сервер
PORT=3333
HOST=localhost
NODE_ENV=development
APP_KEY=<generate-with-node-ace-generate:key>
# База данных PostgreSQL
DB_HOST=127.0.0.1
DB_PORT=5432
DB_USER=postgres
DB_PASSWORD=your_password
DB_DATABASE=parser_zakupok
# Telegram Bot
TELEGRAM_BOT_TOKEN=your_bot_token_from_@BotFather
# Опционально
LOG_LEVEL=info
```
## Команды запуска
### Режим разработки
```bash
# Запуск с hot reload (рекомендуется)
npm run dev
# Альтернативный вариант
node ace serve --hmr
# С watch режимом
node ace serve --watch
```
### Production режим
```bash
# 1. Собрать проект
npm run build
# 2. Запустить миграции
node ace migration:run --force
# 3. Запустить сервер
npm start
# или
node build/bin/server.js
```
## Команды базы данных
### Миграции
```bash
# Выполнить все pending миграции
node ace migration:run
# Откатить последний batch
node ace migration:rollback
# Откатить все миграции
node ace migration:rollback --batch=0
# Пересоздать базу (drop + migrate)
node ace migration:fresh
# Fresh + seed данные
node ace migration:fresh --seed
```
### Создание новых сущностей
```bash
# Создать миграцию
node ace make:migration create_table_name
node ace make:migration add_column_to_table
# Создать модель
node ace make:model ModelName
# Создать модель + миграцию
node ace make:model ModelName -m
# Создать контроллер
node ace make:controller ControllerName
```
### Database утилиты
```bash
# Открыть REPL с доступом к моделям
node ace repl
# Примеры в REPL:
# > await loadModels()
# > const Auction = await import('#models/auction')
# > await Auction.default.all()
```
## Команды парсинга
### Основная команда parse:auctions
```bash
# Парсинг с дефолтными настройками (до 10 страниц)
node ace parse:auctions
# Парсинг конкретного количества страниц
node ace parse:auctions --pages=5
# Парсинг без отправки уведомлений (только сохранение)
node ace parse:auctions --skip-notifications
# Парсинг 1 страницы без уведомлений (для тестирования)
node ace parse:auctions --pages=1 --skip-notifications
# Комбинация опций
node ace parse:auctions --pages=20 --skip-notifications
```
**Что делает команда:**
1. Создает запись в `parse_logs` (статус: running)
2. Парсит страницы с icetrade.by с rate limiting (1 сек между запросами)
3. Валидирует данные через Zod схемы
4. Upsert аукционов по `auction_num` (обновляет существующие, создает новые)
5. Ищет совпадения по ключевым словам
6. Создает записи уведомлений для пользователей
7. Обновляет parse_log (статус: completed/failed, статистика)
**Выводит:**
- Количество найденных аукционов
- Количество новых аукционов
- Количество обновленных аукционов
- Количество созданных уведомлений
- Время выполнения
- Ошибки (если есть)
## Команды Scheduler
### Управление планировщиком
```bash
# Запустить scheduler (production)
node ace scheduler:work
# Запустить с watch режимом (development)
node ace scheduler:work --watch
# Показать список всех запланированных задач
node ace scheduler:list
# Вывод scheduler:list:
# ┌────────────────┬──────────────────┬─────────────────┐
# │ Name │ Cron Expression │ Next Run │
# ├────────────────┼──────────────────┼─────────────────┤
# │ parse:auctions │ 0 0 */6 * * * │ 2024-01-15 18:00│
# └────────────────┴──────────────────┴─────────────────┘
```
### Настройка расписания
Расписание настроено в `start/scheduler.ts`:
```typescript
// Текущее: каждые 6 часов (00:00, 06:00, 12:00, 18:00)
scheduler.call(async () => {
await ace.exec('parse:auctions', [])
}).cron('0 0 */6 * * *')
// Другие варианты расписания:
// .cron('0 0 * * * *') // Каждый час
// .cron('0 */30 * * * *') // Каждые 30 минут
// .everyFourHours() // Каждые 4 часа
// .dailyAt('09:00') // Ежедневно в 9:00
// .twiceDaily(9, 18) // Дважды в день (9:00, 18:00)
```
### Production deployment scheduler
```bash
# Вариант 1: Использовать встроенный scheduler
npm start & # Запустить сервер
node ace scheduler:work # Запустить scheduler в отдельном процессе
# Вариант 2: Использовать PM2 (рекомендуется)
pm2 start ecosystem.config.js
# Вариант 3: Systemd services
# Создать два сервиса: parser-app.service и parser-scheduler.service
```
## Команды тестирования
```bash
# Запустить все тесты
npm test
# или
node ace test
# Запустить конкретный файл теста
node ace test --files=tests/unit/models/auction.spec.ts
# Запустить тесты по паттерну (grep)
node ace test --grep="keyword matching"
node ace test --grep="scraper"
# Запустить только unit тесты
node ace test tests/unit
# Запустить только functional тесты
node ace test tests/functional
```
## Code Quality команды
```bash
# Проверка типов TypeScript
npm run typecheck
# Линтинг (ESLint)
npm run lint
# Автофикс линтинга
npm run lint -- --fix
# Форматирование (Prettier)
npm run format
# Проверить форматирование без изменений
npm run format -- --check
```
## Telegram Bot команды
```bash
# Запустить Telegram бота (когда будет реализовано в Phase 5)
node ace telegram:start
# С watch режимом
node ace telegram:start --watch
```
### Команды бота для пользователей
После запуска бота, пользователи могут использовать:
- `/start` - Регистрация в системе
- `/addkeyword <слово>` - Добавить ключевое слово
- `/keywords` или `/listkeywords` - Список ваших ключевых слов
- `/deletekeyword <id>` - Удалить ключевое слово
- `/help` - Справка по командам
## Структура проекта
```
parser-zakupok/
├── app/
│ ├── controllers/ # HTTP контроллеры
│ ├── models/ # Lucid ORM модели
│ │ ├── auction.ts
│ │ ├── keyword.ts
│ │ ├── user.ts
│ │ ├── notification.ts
│ │ └── parse_log.ts
│ ├── services/ # Бизнес-логика
│ │ ├── scraper_service.ts # Парсинг icetrade.by
│ │ └── notification_service.ts # Обработка уведомлений
│ ├── middleware/ # HTTP middleware
│ └── validators/ # VineJS валидаторы
├── commands/ # Ace CLI команды
│ └── parse_auctions.ts # Команда парсинга
├── config/ # Конфигурация приложения
│ ├── app.ts
│ ├── database.ts
│ └── logger.ts
├── database/
│ └── migrations/ # Database миграции
├── start/
│ ├── routes.ts # Маршруты
│ ├── kernel.ts # Middleware регистрация
│ ├── scheduler.ts # Настройка scheduler
│ └── env.ts # Env validation
├── tests/ # Тесты (Japa)
│ ├── unit/
│ └── functional/
├── docs/ # Документация
├── .env # Environment переменные
├── adonisrc.ts # AdonisJS конфигурация
└── package.json
```
## Import Aliases
В проекте настроены удобные алиасы для импортов:
```typescript
#controllers/* → ./app/controllers/*.js
#models/* → ./app/models/*.js
#services/* → ./app/services/*.js
#validators/* → ./app/validators/*.js
#middleware/* → ./app/middleware/*.js
#config/* → ./config/*.js
#database/* → ./database/*.js
#start/* → ./start/*.js
// Пример использования:
import Auction from '#models/auction'
import ScraperService from '#services/scraper_service'
import { DatabaseConfig } from '#config/database'
```
## Логирование
Все операции логируются через Pino logger:
```typescript
import logger from '@adonisjs/core/services/logger'
// В коде:
logger.info('Scraping started')
logger.error({ err }, 'Failed to parse page')
logger.debug({ count: auctions.length }, 'Auctions scraped')
```
Логи сохраняются в:
- `tmp/logs/app.log` (production)
- Console output (development)
## Parse Logs в БД
Каждый запуск парсинга создает запись в таблице `parse_logs`:
```sql
SELECT * FROM parse_logs ORDER BY started_at DESC LIMIT 5;
-- Поля:
-- id, parse_type, status, started_at, completed_at,
-- items_found, items_new, items_updated, items_failed,
-- error_message, error_details
```
Полезные запросы:
```sql
-- Последние успешные парсинги
SELECT * FROM parse_logs
WHERE status = 'completed'
ORDER BY started_at DESC;
-- Статистика по парсингам
SELECT
COUNT(*) as total_runs,
AVG(items_found) as avg_found,
AVG(items_new) as avg_new,
SUM(items_failed) as total_failed
FROM parse_logs
WHERE parse_type = 'auction';
-- Неудачные парсинги с ошибками
SELECT started_at, error_message
FROM parse_logs
WHERE status = 'failed'
ORDER BY started_at DESC;
```
## Troubleshooting
### База данных не подключается
```bash
# Проверить что PostgreSQL запущен
# Linux/Mac:
sudo systemctl status postgresql
# Windows:
# Проверить через Services (services.msc)
# Проверить подключение
psql -U postgres -h localhost
# Создать базу вручную
psql -U postgres
CREATE DATABASE parser_zakupok;
\q
```
### Ошибки при парсинге
```bash
# Проверить логи
tail -f tmp/logs/app.log
# Проверить parse_logs в БД
node ace repl
> await loadModels()
> const ParseLog = await import('#models/parse_log')
> await ParseLog.default.query().orderBy('started_at', 'desc').first()
# Запустить парсинг одной страницы для отладки
node ace parse:auctions --pages=1 --skip-notifications
```
### Scheduler не запускается
```bash
# Проверить что scheduler провайдер добавлен в adonisrc.ts
cat adonisrc.ts | grep scheduler
# Проверить список задач
node ace scheduler:list
# Запустить вручную для проверки
node ace parse:auctions
```
### TypeScript ошибки
```bash
# Очистить build и пересобрать
rm -rf build/
npm run build
# Проверить типы
npm run typecheck
# Обновить зависимости
npm update
```
## API Endpoints (будут реализованы в Phase 7)
```
GET /api/auctions # Список аукционов (пагинация, фильтры)
GET /api/auctions/:id # Детали аукциона
GET /api/keywords # Ключевые слова пользователя
POST /api/keywords # Добавить ключевое слово
DELETE /api/keywords/:id # Удалить ключевое слово
GET /api/notifications # История уведомлений
```
## Docker (будет реализовано в Phase 8)
```bash
# Сборка и запуск
docker-compose up -d
# Просмотр логов
docker-compose logs -f app
# Остановка
docker-compose down
```
## Roadmap
- [x] Phase 1: Project Setup
- [x] Phase 2: Database Models & Migrations
- [x] Phase 3: ScraperService Implementation
- [x] Phase 4: Scheduler Command
- [ ] Phase 5: Telegram Bot Integration
- [ ] Phase 6: NotificationService Enhancement
- [ ] Phase 7: REST API Endpoints
- [ ] Phase 8: Docker Deployment
## Contributing
1. Fork репозиторий
2. Создать feature branch (`git checkout -b feature/AmazingFeature`)
3. Commit изменения (`git commit -m 'Add some AmazingFeature'`)
4. Push в branch (`git push origin feature/AmazingFeature`)
5. Открыть Pull Request
## License
[MIT License](LICENSE)
## Поддержка
При возникновении проблем:
1. Проверьте [документацию](docs/)
2. Создайте [Issue](../../issues)
3. Проверьте существующие Issues
## Полезные ссылки
- [AdonisJS Documentation](https://docs.adonisjs.com/)
- [Lucid ORM](https://lucid.adonisjs.com/)
- [Grammy Telegram Bot](https://grammy.dev/)
- [PostgreSQL Documentation](https://www.postgresql.org/docs/)

27
ace.js Normal file
View File

@ -0,0 +1,27 @@
/*
|--------------------------------------------------------------------------
| JavaScript entrypoint for running ace commands
|--------------------------------------------------------------------------
|
| DO NOT MODIFY THIS FILE AS IT WILL BE OVERRIDDEN DURING THE BUILD
| PROCESS.
|
| See docs.adonisjs.com/guides/typescript-build-process#creating-production-build
|
| Since, we cannot run TypeScript source code using "node" binary, we need
| a JavaScript entrypoint to run ace commands.
|
| This file registers the "ts-node/esm" hook with the Node.js module system
| and then imports the "bin/console.ts" file.
|
*/
/**
* Register hook to process TypeScript files using ts-node
*/
import 'ts-node-maintained/register/esm'
/**
* Import ace console entrypoint
*/
await import('./bin/console.js')

115
adonisrc.ts Normal file
View File

@ -0,0 +1,115 @@
import { defineConfig } from '@adonisjs/core/app'
export default defineConfig({
/*
|--------------------------------------------------------------------------
| Experimental flags
|--------------------------------------------------------------------------
|
| The following features will be enabled by default in the next major release
| of AdonisJS. You can opt into them today to avoid any breaking changes
| during upgrade.
|
*/
experimental: {
mergeMultipartFieldsAndFiles: true,
shutdownInReverseOrder: true,
},
/*
|--------------------------------------------------------------------------
| Commands
|--------------------------------------------------------------------------
|
| List of ace commands to register from packages. The application commands
| will be scanned automatically from the "./commands" directory.
|
*/
commands: [
() => import('@adonisjs/core/commands'),
() => import('@adonisjs/lucid/commands'),
() => import('adonisjs-scheduler/commands')
],
/*
|--------------------------------------------------------------------------
| Service providers
|--------------------------------------------------------------------------
|
| List of service providers to import and register when booting the
| application
|
*/
providers: [
() => import('@adonisjs/core/providers/app_provider'),
() => import('@adonisjs/core/providers/hash_provider'),
{
file: () => import('@adonisjs/core/providers/repl_provider'),
environment: ['repl', 'test'],
},
() => import('@adonisjs/core/providers/vinejs_provider'),
() => import('@adonisjs/core/providers/edge_provider'),
() => import('@adonisjs/session/session_provider'),
() => import('@adonisjs/vite/vite_provider'),
() => import('@adonisjs/shield/shield_provider'),
() => import('@adonisjs/static/static_provider'),
() => import('@adonisjs/lucid/database_provider'),
() => import('@adonisjs/auth/auth_provider'),
() => import('adonisjs-scheduler/scheduler_provider')
],
/*
|--------------------------------------------------------------------------
| Preloads
|--------------------------------------------------------------------------
|
| List of modules to import before starting the application.
|
*/
preloads: [
() => import('#start/routes'),
() => import('#start/kernel'),
() => import('#start/scheduler')
],
/*
|--------------------------------------------------------------------------
| Tests
|--------------------------------------------------------------------------
|
| List of test suites to organize tests by their type. Feel free to remove
| and add additional suites.
|
*/
tests: {
suites: [
{
files: ['tests/unit/**/*.spec(.ts|.js)'],
name: 'unit',
timeout: 2000,
},
{
files: ['tests/functional/**/*.spec(.ts|.js)'],
name: 'functional',
timeout: 30000,
},
],
forceExit: false,
},
metaFiles: [
{
pattern: 'resources/views/**/*.edge',
reloadServer: false,
},
{
pattern: 'public/**',
reloadServer: false,
},
],
assetsBundler: false,
hooks: {
onBuildStarting: [() => import('@adonisjs/vite/build_hook')],
},
})

View File

@ -0,0 +1,324 @@
import type { HttpContext } from '@adonisjs/core/http'
import Auction from '#models/auction'
import ParseLog from '#models/parse_log'
import { errors as lucidErrors } from '@adonisjs/lucid'
import { DateTime } from 'luxon'
import { ScraperService } from '#services/scraper_service'
import { NotificationService } from '#services/notification_service'
import logger from '@adonisjs/core/services/logger'
export default class AuctionsController {
/**
* List auctions with pagination
* GET /api/auctions?page=1&limit=20
*/
async index({ request, response }: HttpContext) {
const page = request.input('page', 1)
const limit = request.input('limit', 20)
const pageNumber = Math.max(1, Number(page))
const limitNumber = Math.min(100, Math.max(1, Number(limit)))
try {
const auctions = await Auction.query()
.orderBy('created_at', 'desc')
.paginate(pageNumber, limitNumber)
return response.ok(auctions.serialize())
} catch (error) {
return response.internalServerError({
error: 'Failed to fetch auctions',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
/**
* Get a single auction by id with relationships
* GET /api/auctions/:id
*/
async show({ params, response }: HttpContext) {
const auctionId = params.id
if (!auctionId || isNaN(Number(auctionId))) {
return response.badRequest({
error: 'Invalid auction id',
})
}
try {
const auction = await Auction.query()
.where('id', auctionId)
.preload('notifications')
.firstOrFail()
return response.ok(auction.serialize())
} catch (error) {
if (error instanceof lucidErrors.E_ROW_NOT_FOUND) {
return response.notFound({
error: 'Auction not found',
})
}
return response.internalServerError({
error: 'Failed to fetch auction',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
/**
* Search auctions by keyword/title/status
* GET /api/auctions/search?q=keyword&status=active&page=1&limit=20
*/
async search({ request, response }: HttpContext) {
const searchQuery = request.input('q', '')
const status = request.input('status')
const page = request.input('page', 1)
const limit = request.input('limit', 20)
const pageNumber = Math.max(1, Number(page))
const limitNumber = Math.min(100, Math.max(1, Number(limit)))
try {
const query = Auction.query()
if (searchQuery && typeof searchQuery === 'string' && searchQuery.trim().length > 0) {
const trimmedQuery = searchQuery.trim()
query.where((builder) => {
builder
.whereILike('title', `%${trimmedQuery}%`)
.orWhereILike('description', `%${trimmedQuery}%`)
.orWhereILike('auction_num', `%${trimmedQuery}%`)
.orWhereILike('organization', `%${trimmedQuery}%`)
})
}
if (status && typeof status === 'string' && status.trim().length > 0) {
query.where('status', status.trim())
}
const auctions = await query.orderBy('created_at', 'desc').paginate(pageNumber, limitNumber)
return response.ok(auctions.serialize())
} catch (error) {
return response.internalServerError({
error: 'Failed to search auctions',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
/**
* List all auctions from last 3 days
* GET /list
*/
async list({ response }: HttpContext) {
try {
const threeDaysAgo = DateTime.now().minus({ days: 3 })
const auctions = await Auction.query()
.where('created_at', '>=', threeDaysAgo.toSQL())
.orderBy('created_at', 'desc')
return response.ok({
data: auctions.map((auction) => auction.serialize()),
meta: {
total: auctions.length,
from_date: threeDaysAgo.toISO(),
},
})
} catch (error) {
return response.internalServerError({
error: 'Failed to fetch recent auctions',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
/**
* Render HTML view of auctions from last 3 days
* GET /list-view
*/
async listView({ view }: HttpContext) {
const threeDaysAgo = DateTime.now().minus({ days: 3 })
const auctions = await Auction.query()
.where('created_at', '>=', threeDaysAgo.toSQL())
.orderBy('created_at', 'desc')
return view.render('auctions/list', {
auctions,
fromDate: threeDaysAgo.toFormat('dd.MM.yyyy HH:mm'),
})
}
/**
* Trigger auction parsing from web interface
* POST /trigger-parse
*/
async triggerParse({ request, response, session }: HttpContext) {
try {
// Validate input
const pages = request.input('pages', 1)
const notifySubscribers = request.input('notifySubscribers') === 'on'
const pagesNumber = Math.max(1, Math.min(10, Number(pages)))
if (isNaN(pagesNumber)) {
session.flash('error', 'Invalid number of pages')
return response.redirect('/list-view')
}
logger.info(`Parsing triggered from web interface`, {
pages: pagesNumber,
notifySubscribers,
})
const startTime = DateTime.now()
const scraper = new ScraperService()
const notificationService = new NotificationService()
// Create ParseLog entry
const parseLog = await ParseLog.create({
parseType: 'auction',
status: 'running',
itemsFound: 0,
errors: null,
startedAt: startTime,
completedAt: null,
})
let newAuctions = 0
let updatedAuctions = 0
let totalScraped = 0
const errors: string[] = []
// Step 1: Scrape auctions
logger.info(`Scraping ${pagesNumber} page(s) from icetrade.by`)
const auctionData = await scraper.scrapeAuctions(pagesNumber)
totalScraped = auctionData.length
if (totalScraped === 0) {
await this.updateParseLog(parseLog, 'completed', 0, null)
session.flash('error', 'No auctions found during parsing')
return response.redirect('/list-view')
}
// Step 2: Upsert auctions to database
logger.info(`Saving ${totalScraped} auction(s) to database`)
for (const data of auctionData) {
try {
// Check if auction exists by auctionNum
const existingAuction = await Auction.query()
.where('auctionNum', data.auctionNum)
.first()
let auction: Auction
if (existingAuction) {
// Update existing auction
existingAuction.title = data.title
existingAuction.description = data.description
existingAuction.organization = data.organization
existingAuction.status = data.status
existingAuction.deadline = data.deadline ? DateTime.fromISO(data.deadline) : null
existingAuction.url = data.link
existingAuction.rawData = data
await existingAuction.save()
auction = existingAuction
updatedAuctions++
logger.debug(`Updated auction: ${data.auctionNum}`)
} else {
// Create new auction
auction = await Auction.create({
auctionNum: data.auctionNum,
title: data.title,
description: data.description,
organization: data.organization,
status: data.status,
price: null,
deadline: data.deadline ? DateTime.fromISO(data.deadline) : null,
url: data.link,
rawData: data,
})
newAuctions++
logger.debug(`Created auction: ${data.auctionNum}`)
// Step 3: Check for keyword matches and send notifications (for new auctions only)
if (notifySubscribers) {
try {
await notificationService.checkAndNotify(auction)
} catch (error) {
const errorMsg = `Notification check failed for auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
logger.error(errorMsg)
errors.push(errorMsg)
// Don't fail the entire process if notification fails
}
}
}
} catch (error) {
const errorMsg = `Failed to save auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
logger.error(errorMsg)
errors.push(errorMsg)
}
}
// Update ParseLog with success status
const status = errors.length > 0 ? 'completed_with_errors' : 'completed'
await this.updateParseLog(
parseLog,
status,
newAuctions + updatedAuctions,
errors.length > 0 ? errors.join('\n') : null
)
// Create success message
let successMessage = `Parsing completed: ${totalScraped} auctions scraped, ${newAuctions} new, ${updatedAuctions} updated`
if (!notifySubscribers) {
successMessage += ' (notifications disabled)'
}
if (errors.length > 0) {
successMessage += ` with ${errors.length} error(s)`
}
session.flash('success', successMessage)
logger.info(successMessage)
return response.redirect('/list-view')
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error during parsing'
logger.error('Error in triggerParse', {
error: errorMessage,
})
session.flash('error', `Parsing failed: ${errorMessage}`)
return response.redirect('/list-view')
}
}
/**
* Update ParseLog entry with final status
*/
private async updateParseLog(
parseLog: ParseLog,
status: string,
itemsFound: number,
errors: string | null
): Promise<void> {
parseLog.status = status
parseLog.itemsFound = itemsFound
parseLog.errors = errors
parseLog.completedAt = DateTime.now()
await parseLog.save()
logger.info(`Updated ParseLog ${parseLog.id}: status=${status}, items=${itemsFound}`)
}
}

View File

@ -0,0 +1,101 @@
import type { HttpContext } from '@adonisjs/core/http'
import Keyword from '#models/keyword'
export default class KeywordsController {
/**
* List keywords by user_id with optional filtering by is_active
* GET /api/keywords?user_id=1&is_active=true
*/
async index({ request, response }: HttpContext) {
const userId = request.input('user_id')
const isActive = request.input('is_active')
if (!userId) {
return response.badRequest({
error: 'user_id query parameter is required',
})
}
const query = Keyword.query().where('user_id', userId)
if (isActive !== undefined) {
const activeValue = isActive === 'true' || isActive === true
query.where('is_active', activeValue)
}
const keywords = await query.orderBy('created_at', 'desc')
return response.ok(keywords)
}
/**
* Create a new keyword
* POST /api/keywords
* Body: { user_id: number, keyword: string, case_sensitive?: boolean }
*/
async store({ request, response }: HttpContext) {
const userId = request.input('user_id')
const keyword = request.input('keyword')
const caseSensitive = request.input('case_sensitive', false)
if (!userId || !keyword) {
return response.badRequest({
error: 'user_id and keyword are required',
})
}
if (typeof keyword !== 'string' || keyword.trim().length === 0) {
return response.badRequest({
error: 'keyword must be a non-empty string',
})
}
try {
const newKeyword = await Keyword.create({
userId,
keyword: keyword.trim(),
caseSensitive,
isActive: true,
})
return response.created(newKeyword)
} catch (error) {
return response.internalServerError({
error: 'Failed to create keyword',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
/**
* Delete a keyword by id
* DELETE /api/keywords/:id
*/
async destroy({ params, response }: HttpContext) {
const keywordId = params.id
if (!keywordId || isNaN(Number(keywordId))) {
return response.badRequest({
error: 'Invalid keyword id',
})
}
try {
const keyword = await Keyword.findOrFail(keywordId)
await keyword.delete()
return response.noContent()
} catch (error) {
if (error.code === 'E_ROW_NOT_FOUND') {
return response.notFound({
error: 'Keyword not found',
})
}
return response.internalServerError({
error: 'Failed to delete keyword',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
}

View File

@ -0,0 +1,29 @@
import type { HttpContext } from '@adonisjs/core/http'
import ParseLog from '#models/parse_log'
export default class ParseLogsController {
/**
* List parse logs with pagination, ordered by latest
* GET /api/parse-logs?page=1&limit=20
*/
async index({ request, response }: HttpContext) {
const page = request.input('page', 1)
const limit = request.input('limit', 20)
const pageNumber = Math.max(1, Number(page))
const limitNumber = Math.min(100, Math.max(1, Number(limit)))
try {
const parseLogs = await ParseLog.query()
.orderBy('started_at', 'desc')
.paginate(pageNumber, limitNumber)
return response.ok(parseLogs.serialize())
} catch (error) {
return response.internalServerError({
error: 'Failed to fetch parse logs',
message: error instanceof Error ? error.message : 'Unknown error',
})
}
}
}

49
app/exceptions/handler.ts Normal file
View File

@ -0,0 +1,49 @@
import app from '@adonisjs/core/services/app'
import { HttpContext, ExceptionHandler } from '@adonisjs/core/http'
import type { StatusPageRange, StatusPageRenderer } from '@adonisjs/core/types/http'
export default class HttpExceptionHandler extends ExceptionHandler {
/**
* In debug mode, the exception handler will display verbose errors
* with pretty printed stack traces.
*/
protected debug = !app.inProduction
/**
* Status pages are used to display a custom HTML pages for certain error
* codes. You might want to enable them in production only, but feel
* free to enable them in development as well.
*/
protected renderStatusPages = app.inProduction
/**
* Status pages is a collection of error code range and a callback
* to return the HTML contents to send as a response.
*/
protected statusPages: Record<StatusPageRange, StatusPageRenderer> = {
'404': (error, { view }) => {
return view.render('pages/errors/not_found', { error })
},
'500..599': (error, { view }) => {
return view.render('pages/errors/server_error', { error })
},
}
/**
* The method is used for handling errors and returning
* response to the client
*/
async handle(error: unknown, ctx: HttpContext) {
return super.handle(error, ctx)
}
/**
* The method is used to report error to the logging service or
* the a third party error monitoring service.
*
* @note You should not attempt to send a response from this method.
*/
async report(error: unknown, ctx: HttpContext) {
return super.report(error, ctx)
}
}

View File

@ -0,0 +1,25 @@
import type { HttpContext } from '@adonisjs/core/http'
import type { NextFn } from '@adonisjs/core/types/http'
import type { Authenticators } from '@adonisjs/auth/types'
/**
* Auth middleware is used authenticate HTTP requests and deny
* access to unauthenticated users.
*/
export default class AuthMiddleware {
/**
* The URL to redirect to, when authentication fails
*/
redirectTo = '/login'
async handle(
ctx: HttpContext,
next: NextFn,
options: {
guards?: (keyof Authenticators)[]
} = {}
) {
await ctx.auth.authenticateUsing(options.guards, { loginRoute: this.redirectTo })
return next()
}
}

View File

@ -0,0 +1,19 @@
import { Logger } from '@adonisjs/core/logger'
import { HttpContext } from '@adonisjs/core/http'
import { NextFn } from '@adonisjs/core/types/http'
/**
* The container bindings middleware binds classes to their request
* specific value using the container resolver.
*
* - We bind "HttpContext" class to the "ctx" object
* - And bind "Logger" class to the "ctx.logger" object
*/
export default class ContainerBindingsMiddleware {
handle(ctx: HttpContext, next: NextFn) {
ctx.containerResolver.bindValue(HttpContext, ctx)
ctx.containerResolver.bindValue(Logger, ctx.logger)
return next()
}
}

View File

@ -0,0 +1,31 @@
import type { HttpContext } from '@adonisjs/core/http'
import type { NextFn } from '@adonisjs/core/types/http'
import type { Authenticators } from '@adonisjs/auth/types'
/**
* Guest middleware is used to deny access to routes that should
* be accessed by unauthenticated users.
*
* For example, the login page should not be accessible if the user
* is already logged-in
*/
export default class GuestMiddleware {
/**
* The URL to redirect to when user is logged-in
*/
redirectTo = '/'
async handle(
ctx: HttpContext,
next: NextFn,
options: { guards?: (keyof Authenticators)[] } = {}
) {
for (let guard of options.guards || [ctx.auth.defaultGuard]) {
if (await ctx.auth.use(guard).check()) {
return ctx.response.redirect(this.redirectTo, true)
}
}
return next()
}
}

View File

@ -0,0 +1,19 @@
import type { HttpContext } from '@adonisjs/core/http'
import type { NextFn } from '@adonisjs/core/types/http'
/**
* Silent auth middleware can be used as a global middleware to silent check
* if the user is logged-in or not.
*
* The request continues as usual, even when the user is not logged-in.
*/
export default class SilentAuthMiddleware {
async handle(
ctx: HttpContext,
next: NextFn,
) {
await ctx.auth.check()
return next()
}
}

48
app/models/auction.ts Normal file
View File

@ -0,0 +1,48 @@
import { DateTime } from 'luxon'
import { BaseModel, column, hasMany } from '@adonisjs/lucid/orm'
import type { HasMany } from '@adonisjs/lucid/types/relations'
import Notification from '#models/notification'
export default class Auction extends BaseModel {
@column({ isPrimary: true })
declare id: number
@column()
declare auctionNum: string
@column()
declare title: string
@column()
declare description: string | null
@column()
declare organization: string | null
@column()
declare status: string | null
@column()
declare price: number | null
@column.dateTime()
declare deadline: DateTime | null
@column()
declare url: string | null
@column({
prepare: (value: Record<string, any> | null) => (value ? JSON.stringify(value) : null),
consume: (value: string | null) => (value ? JSON.parse(value) : null),
})
declare rawData: Record<string, any> | null
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime
@hasMany(() => Notification)
declare notifications: HasMany<typeof Notification>
}

30
app/models/keyword.ts Normal file
View File

@ -0,0 +1,30 @@
import { DateTime } from 'luxon'
import { BaseModel, column, belongsTo } from '@adonisjs/lucid/orm'
import type { BelongsTo } from '@adonisjs/lucid/types/relations'
import User from '#models/user'
export default class Keyword extends BaseModel {
@column({ isPrimary: true })
declare id: number
@column()
declare userId: number
@column()
declare keyword: string
@column()
declare isActive: boolean
@column()
declare caseSensitive: boolean
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime
@belongsTo(() => User)
declare user: BelongsTo<typeof User>
}

View File

@ -0,0 +1,37 @@
import { DateTime } from 'luxon'
import { BaseModel, column, belongsTo } from '@adonisjs/lucid/orm'
import type { BelongsTo } from '@adonisjs/lucid/types/relations'
import Auction from '#models/auction'
import Keyword from '#models/keyword'
export default class Notification extends BaseModel {
@column({ isPrimary: true })
declare id: number
@column()
declare auctionId: number
@column()
declare keywordId: number
@column()
declare status: 'pending' | 'sent' | 'failed'
@column()
declare errorMessage: string | null
@column.dateTime()
declare sentAt: DateTime | null
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime
@belongsTo(() => Auction)
declare auction: BelongsTo<typeof Auction>
@belongsTo(() => Keyword)
declare keyword: BelongsTo<typeof Keyword>
}

31
app/models/parse_log.ts Normal file
View File

@ -0,0 +1,31 @@
import { DateTime } from 'luxon'
import { BaseModel, column } from '@adonisjs/lucid/orm'
export default class ParseLog extends BaseModel {
@column({ isPrimary: true })
declare id: number
@column()
declare parseType: string
@column()
declare status: string
@column()
declare itemsFound: number
@column()
declare errors: string | null
@column.dateTime()
declare startedAt: DateTime
@column.dateTime()
declare completedAt: DateTime | null
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime
}

30
app/models/user.ts Normal file
View File

@ -0,0 +1,30 @@
import { DateTime } from 'luxon'
import hash from '@adonisjs/core/services/hash'
import { compose } from '@adonisjs/core/helpers'
import { BaseModel, column } from '@adonisjs/lucid/orm'
import { withAuthFinder } from '@adonisjs/auth/mixins/lucid'
const AuthFinder = withAuthFinder(() => hash.use('scrypt'), {
uids: ['email'],
passwordColumnName: 'password',
})
export default class User extends compose(BaseModel, AuthFinder) {
@column({ isPrimary: true })
declare id: number
@column()
declare email: string
@column({ serializeAs: null })
declare password: string
@column()
declare telegramChatId: string | null
@column.dateTime({ autoCreate: true })
declare createdAt: DateTime
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updatedAt: DateTime
}

View File

@ -0,0 +1,20 @@
import { z } from 'zod'
/**
* Zod schema for auction data validation
* Ensures all scraped data conforms to expected structure before database insertion
*/
export const AuctionSchema = z.object({
auctionNum: z.string().trim().min(1, 'Auction number is required'),
title: z.string().trim().min(1, 'Title is required'),
organization: z.string().trim().min(1, 'Organization is required'),
status: z.string().trim().min(1, 'Status is required'),
deadline: z.string().nullable().default(null),
link: z.string().url('Link must be a valid URL'),
description: z.string().nullable().default(null),
})
/**
* TypeScript type inferred from schema
*/
export type AuctionData = z.infer<typeof AuctionSchema>

View File

@ -0,0 +1,223 @@
import logger from '@adonisjs/core/services/logger'
import { DateTime } from 'luxon'
import Auction from '#models/auction'
import Keyword from '#models/keyword'
import Notification from '#models/notification'
import User from '#models/user'
import { TelegramService } from '#services/telegram_service'
/**
* NotificationService - Handles keyword matching and notification delivery
*
* Features:
* - Matches keywords against auction titles and descriptions
* - Case-sensitive and case-insensitive matching support
* - Prevents duplicate notifications (checks existing records)
* - Tracks notification status (pending/sent/failed)
* - Integrates with TelegramService for message delivery
* - Validates user active status before sending
*/
export class NotificationService {
/**
* Check auction against all active keywords and send notifications for matches
*
* @param auction - The auction to check for keyword matches
*/
async checkAndNotify(auction: Auction): Promise<void> {
try {
logger.info(`Checking auction ${auction.auctionNum} for keyword matches`)
// Get all active keywords
const keywords = await Keyword.query().where('isActive', true).preload('user')
if (keywords.length === 0) {
logger.debug('No active keywords found, skipping notification check')
return
}
logger.debug(`Found ${keywords.length} active keyword(s) to check`)
let matchCount = 0
for (const keyword of keywords) {
try {
// Check if keyword matches auction title or description
const titleMatch = this.matchKeyword(
auction.title,
keyword.keyword,
keyword.caseSensitive
)
const descMatch = auction.description
? this.matchKeyword(auction.description, keyword.keyword, keyword.caseSensitive)
: false
if (!titleMatch && !descMatch) {
continue // No match, skip this keyword
}
matchCount++
logger.info(
`Keyword match found: "${keyword.keyword}" in auction ${auction.auctionNum}`,
{
keywordId: keyword.id,
auctionId: auction.id,
matchedIn: titleMatch ? 'title' : 'description',
}
)
// Check if notification already exists (prevent duplicates)
const existingNotification = await Notification.query()
.where('auctionId', auction.id)
.where('keywordId', keyword.id)
.first()
if (existingNotification) {
logger.debug(
`Notification already exists for auction ${auction.id} and keyword ${keyword.id}, skipping`,
{
notificationId: existingNotification.id,
status: existingNotification.status,
}
)
continue
}
// Create notification record with 'pending' status
const notification = await Notification.create({
auctionId: auction.id,
keywordId: keyword.id,
status: 'pending',
errorMessage: null,
sentAt: null,
})
logger.debug(`Created notification record ${notification.id}`)
// Get user and validate active status
const user = await User.find(keyword.userId)
if (!user) {
logger.warn(`User ${keyword.userId} not found for keyword ${keyword.id}`)
await notification
.merge({
status: 'failed',
errorMessage: 'User not found',
})
.save()
continue
}
if (!user.telegramChatId) {
logger.warn(`User ${user.id} has no Telegram chat ID`)
await notification
.merge({
status: 'failed',
errorMessage: 'User has no Telegram chat ID',
})
.save()
continue
}
// Send notification via Telegram
try {
const telegramService = TelegramService.getInstance()
const auctionUrl = auction.url || `https://icetrade.by/auction/${auction.auctionNum}`
const success = await telegramService.sendNotification(
user.telegramChatId,
auction.title,
auctionUrl,
keyword.keyword
)
if (success) {
// Update notification status to 'sent'
await notification
.merge({
status: 'sent',
sentAt: DateTime.now(),
})
.save()
logger.info(`Notification sent successfully`, {
notificationId: notification.id,
userId: user.id,
chatId: user.telegramChatId,
})
} else {
// Update notification status to 'failed'
await notification
.merge({
status: 'failed',
errorMessage: 'Failed to send Telegram message (unknown error)',
})
.save()
logger.error(`Failed to send notification`, {
notificationId: notification.id,
userId: user.id,
chatId: user.telegramChatId,
})
}
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : 'Unknown error during notification send'
logger.error(`Error sending notification`, {
notificationId: notification.id,
error: errorMessage,
userId: user.id,
})
// Update notification status to 'failed' with error details
await notification
.merge({
status: 'failed',
errorMessage,
})
.save()
}
} catch (error) {
logger.error(`Error processing keyword ${keyword.id} for auction ${auction.id}`, {
error: error instanceof Error ? error.message : String(error),
keywordId: keyword.id,
auctionId: auction.id,
})
// Continue processing other keywords even if one fails
continue
}
}
logger.info(
`Completed notification check for auction ${auction.auctionNum}: ${matchCount} match(es) found`
)
} catch (error) {
logger.error(`Error in checkAndNotify for auction ${auction.id}`, {
error: error instanceof Error ? error.message : String(error),
auctionId: auction.id,
})
// Don't throw - allow scraper to continue processing other auctions
}
}
/**
* Check if a keyword matches text with case-sensitive or case-insensitive matching
*
* @param text - The text to search in
* @param keyword - The keyword to search for
* @param caseSensitive - Whether to perform case-sensitive matching
* @returns true if keyword is found in text, false otherwise
*/
private matchKeyword(text: string, keyword: string, caseSensitive: boolean): boolean {
if (!text || !keyword) {
return false
}
if (caseSensitive) {
return text.includes(keyword)
}
return text.toLowerCase().includes(keyword.toLowerCase())
}
}

View File

@ -0,0 +1,338 @@
import * as cheerio from 'cheerio'
import logger from '@adonisjs/core/services/logger'
import { AuctionSchema, type AuctionData } from '../schemas/auction_schema.js'
/**
* Configuration for the scraper
*/
const SCRAPER_CONFIG = {
baseUrl: 'https://icetrade.by',
requestDelay: 1000, // 1 second between requests
timeout: 30000, // 30 seconds timeout
userAgent:
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
maxRetries: 3,
retryDelay: 2000,
} as const
/**
* Error thrown when scraping fails
*/
export class ScraperError extends Error {
constructor(
message: string,
public readonly cause?: unknown
) {
super(message)
this.name = 'ScraperError'
}
}
/**
* ScraperService - Handles fetching and parsing auction data from icetrade.by
*
* Features:
* - Rate limiting (1s delay between requests)
* - Retry logic with exponential backoff
* - Comprehensive error handling
* - Data validation with Zod
* - Structured logging
*/
export class ScraperService {
/**
* Builds the URL for fetching auctions with all required parameters
*/
private buildUrl(pageNumber: number): string {
const params = new URLSearchParams({
search_text: '',
'zakup_type[1]': '1',
'zakup_type[2]': '1',
onPage: '100',
sort: 'num:desc',
p: pageNumber.toString(),
// Status flags
'r[1]': '1',
'r[2]': '1',
'r[3]': '1',
'r[4]': '1',
'r[5]': '1',
'r[6]': '1',
'r[7]': '1',
// Trade type flags
't[Trade]': '1',
't[contest]': '1',
't[request]': '1',
't[qualification]': '1',
't[negotiations]': '1',
})
return `${SCRAPER_CONFIG.baseUrl}/trades/index?${params.toString()}`
}
/**
* Fetches HTML content from the specified page with retry logic
*/
async fetchPage(pageNumber: number): Promise<string> {
const url = this.buildUrl(pageNumber)
let lastError: Error | undefined
logger.info(`Preparing to fetch URL: ${url}`)
for (let attempt = 1; attempt <= SCRAPER_CONFIG.maxRetries; attempt++) {
try {
logger.info(
`Fetching page ${pageNumber} (attempt ${attempt}/${SCRAPER_CONFIG.maxRetries})`,
{ url }
)
const controller = new AbortController()
const timeoutId = setTimeout(() => controller.abort(), SCRAPER_CONFIG.timeout)
const response = await fetch(url, {
headers: {
'User-Agent': SCRAPER_CONFIG.userAgent,
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
Connection: 'keep-alive',
},
signal: controller.signal,
})
clearTimeout(timeoutId)
logger.info(`Received response: status=${response.status} ${response.statusText}`)
if (!response.ok) {
const bodyText = await response.text().catch(() => 'Unable to read response body')
throw new Error(
`HTTP ${response.status}: ${response.statusText}. Body: ${bodyText.substring(0, 200)}`
)
}
const html = await response.text()
if (!html || html.trim().length === 0) {
throw new Error('Received empty response')
}
logger.info(`Successfully fetched page ${pageNumber}: ${html.length} bytes`)
return html
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error))
// Log detailed error information
const errorDetails: Record<string, any> = {
message: lastError.message,
name: lastError.name,
url,
}
// Add stack trace for non-HTTP errors
if (!(lastError.message.startsWith('HTTP '))) {
errorDetails.stack = lastError.stack
}
// Check for specific error types
if (lastError.name === 'AbortError') {
errorDetails.reason = 'Request timeout after 30s'
} else if (lastError.message.includes('fetch failed')) {
errorDetails.reason = 'Network error - check DNS, firewall, or connectivity'
} else if (lastError.message.includes('ENOTFOUND')) {
errorDetails.reason = 'DNS resolution failed - domain not found'
} else if (lastError.message.includes('ECONNREFUSED')) {
errorDetails.reason = 'Connection refused - server not reachable'
} else if (lastError.message.includes('ETIMEDOUT')) {
errorDetails.reason = 'Connection timeout - server too slow or unreachable'
}
logger.warn(errorDetails, `Failed to fetch page ${pageNumber} (attempt ${attempt}/${SCRAPER_CONFIG.maxRetries})`)
if (attempt < SCRAPER_CONFIG.maxRetries) {
const delay = SCRAPER_CONFIG.retryDelay * attempt
logger.info(`Retrying in ${delay}ms...`)
await this.delay(delay)
}
}
}
const finalError = new ScraperError(
`Failed to fetch page ${pageNumber} after ${SCRAPER_CONFIG.maxRetries} attempts: ${lastError?.message}`,
lastError
)
logger.error('All fetch attempts failed', {
pageNumber,
url,
lastErrorMessage: lastError?.message,
lastErrorName: lastError?.name,
})
throw finalError
}
/**
* Parses HTML content and extracts auction data
*/
parsePage(html: string): AuctionData[] {
try {
const $ = cheerio.load(html)
const auctions: AuctionData[] = []
// Find the auctions table
const auctionsTable = $('table.auctions.w100')
if (auctionsTable.length === 0) {
logger.warn('No auctions table found in HTML')
return []
}
// Parse each auction row
const rows = auctionsTable.find('tbody tr')
logger.info(`Found ${rows.length} auction rows to parse`)
rows.each((index, element) => {
try {
const row = $(element)
// Extract auction data from table cells
const cells = row.find('td')
if (cells.length < 4) {
logger.warn(`Row ${index} has insufficient cells, skipping`)
return
}
// Extract auction number (typically in first cell)
const auctionNumCell = $(cells[0])
const auctionNum = auctionNumCell.text().trim()
// Extract title and link (typically in second cell with <a> tag)
const titleCell = $(cells[1])
const titleLink = titleCell.find('a').first()
const title = titleLink.text().trim()
const link = titleLink.attr('href')?.trim() || ''
// Skip if link is empty or missing
if (!link) {
logger.warn(`Row ${index} has missing or empty link, skipping`)
return
}
// Make link absolute if it's relative
const absoluteLink = link.startsWith('http')
? link
: `${SCRAPER_CONFIG.baseUrl}${link.startsWith('/') ? link : `/${link}`}`
// Extract organization (typically in third cell)
const organizationCell = $(cells[2])
const organization = organizationCell.text().trim()
// Extract status (typically in fourth cell)
const statusCell = $(cells[3])
const status = statusCell.text().trim()
// Extract deadline if available (typically in fifth cell)
const deadlineCell = $(cells[4])
const deadline = deadlineCell.text().trim() || null
// Description can be extracted from title cell's additional text or separate element
const description = titleCell.find('.description').text().trim() || null
// Validate with Zod schema
const result = AuctionSchema.safeParse({
auctionNum,
title,
organization,
status,
deadline,
link: absoluteLink,
description,
})
if (!result.success) {
logger.warn(`Validation failed for auction at row ${index}`, {
errors: result.error.issues,
data: { auctionNum, title },
})
return
}
auctions.push(result.data)
} catch (error) {
logger.error(`Error parsing auction row ${index}`, {
error: error instanceof Error ? error.message : String(error),
})
}
})
logger.info(`Successfully parsed ${auctions.length} valid auctions`)
return auctions
} catch (error) {
throw new ScraperError(
'Failed to parse HTML content',
error instanceof Error ? error : new Error(String(error))
)
}
}
/**
* Scrapes multiple pages of auctions with rate limiting
*
* @param maxPages - Maximum number of pages to scrape (default: 1)
* @returns Array of all parsed auction data
*/
async scrapeAuctions(maxPages: number = 1): Promise<AuctionData[]> {
if (maxPages < 1) {
throw new Error('maxPages must be at least 1')
}
logger.info(`Starting scrape of ${maxPages} page(s)`)
const allAuctions: AuctionData[] = []
for (let page = 1; page <= maxPages; page++) {
try {
// Fetch page HTML
const html = await this.fetchPage(page)
// Parse auctions from HTML
const auctions = this.parsePage(html)
allAuctions.push(...auctions)
logger.info(`Page ${page}/${maxPages}: Found ${auctions.length} auctions`)
// Rate limiting: wait before next request (except for last page)
if (page < maxPages) {
logger.debug(`Waiting ${SCRAPER_CONFIG.requestDelay}ms before next request`)
await this.delay(SCRAPER_CONFIG.requestDelay)
}
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error))
logger.error(`Error scraping page ${page}`, {
message: err.message,
name: err.name,
stack: err.stack,
cause: err.cause,
})
// Continue to next page instead of failing completely
// This ensures partial data is still returned
continue
}
}
logger.info(
`Scraping completed: ${allAuctions.length} total auctions from ${maxPages} page(s)`
)
return allAuctions
}
/**
* Helper method for delays (rate limiting, retries)
*/
private delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms))
}
}

View File

@ -0,0 +1,404 @@
import { Bot, Context } from 'grammy'
import env from '#start/env'
import logger from '@adonisjs/core/services/logger'
import User from '#models/user'
import Keyword from '#models/keyword'
/**
* TelegramService - Handles Telegram bot operations
*
* Singleton service for managing Grammy bot instance and commands.
* Provides keyword management and notification delivery.
*/
export class TelegramService {
private static instance: TelegramService | null = null
private bot: Bot
private isRunning: boolean = false
private constructor() {
const token = env.get('TELEGRAM_BOT_TOKEN')
this.bot = new Bot(token)
this.setupCommands()
}
/**
* Get singleton instance of TelegramService
*/
static getInstance(): TelegramService {
if (!TelegramService.instance) {
TelegramService.instance = new TelegramService()
}
return TelegramService.instance
}
/**
* Setup bot commands and handlers
*/
private setupCommands(): void {
// /start - Register user
this.bot.command('start', async (ctx) => {
try {
await this.handleStart(ctx)
} catch (error) {
logger.error('Error handling /start command', {
error: error instanceof Error ? error.message : String(error),
chatId: ctx.chat?.id,
})
await ctx.reply('Произошла ошибка при регистрации. Попробуйте позже.')
}
})
// /addkeyword - Add keyword
this.bot.command('addkeyword', async (ctx) => {
try {
await this.handleAddKeyword(ctx)
} catch (error) {
logger.error('Error handling /addkeyword command', {
error: error instanceof Error ? error.message : String(error),
chatId: ctx.chat?.id,
})
await ctx.reply('Произошла ошибка при добавлении ключевого слова. Попробуйте позже.')
}
})
// /listkeywords - List keywords
this.bot.command('listkeywords', async (ctx) => {
try {
await this.handleListKeywords(ctx)
} catch (error) {
logger.error('Error handling /listkeywords command', {
error: error instanceof Error ? error.message : String(error),
chatId: ctx.chat?.id,
})
await ctx.reply('Произошла ошибка при получении списка ключевых слов. Попробуйте позже.')
}
})
// /deletekeyword - Delete keyword
this.bot.command('deletekeyword', async (ctx) => {
try {
await this.handleDeleteKeyword(ctx)
} catch (error) {
logger.error('Error handling /deletekeyword command', {
error: error instanceof Error ? error.message : String(error),
chatId: ctx.chat?.id,
})
await ctx.reply('Произошла ошибка при удалении ключевого слова. Попробуйте позже.')
}
})
// /help - Show help
this.bot.command('help', async (ctx) => {
await this.handleHelp(ctx)
})
// Error handler
this.bot.catch((err) => {
logger.error('Grammy bot error', {
error: err.error instanceof Error ? err.error.message : String(err.error),
ctx: err.ctx,
})
})
}
/**
* Handle /start command - Register user
*/
private async handleStart(ctx: Context): Promise<void> {
if (!ctx.chat?.id) {
await ctx.reply('Не удалось определить chat ID.')
return
}
const chatId = String(ctx.chat.id)
// Check if user already exists
const existingUser = await User.findBy('telegramChatId', chatId)
if (existingUser) {
await ctx.reply(
'Вы уже зарегистрированы!\n\n' +
'Используйте /addkeyword для добавления ключевых слов.\n' +
'Используйте /help для получения справки.'
)
logger.info(`User already registered: ${chatId}`)
return
}
// Create new user
const user = await User.create({
email: `telegram_${chatId}@temp.local`,
password: Math.random().toString(36).substring(2, 15),
telegramChatId: chatId,
})
await ctx.reply(
'Добро пожаловать! Вы успешно зарегистрированы.\n\n' +
'Теперь вы можете добавлять ключевые слова для отслеживания аукционов:\n' +
'/addkeyword <слово> - добавить ключевое слово\n' +
'/listkeywords - список ваших ключевых слов\n' +
'/deletekeyword <id> - удалить ключевое слово\n' +
'/help - справка'
)
logger.info(`New user registered: ${chatId} (user_id: ${user.id})`)
}
/**
* Handle /addkeyword command - Add keyword for user
*/
private async handleAddKeyword(ctx: Context): Promise<void> {
if (!ctx.chat?.id) {
await ctx.reply('Не удалось определить chat ID.')
return
}
const chatId = String(ctx.chat.id)
const user = await User.findBy('telegramChatId', chatId)
if (!user) {
await ctx.reply('Вы не зарегистрированы. Используйте /start для регистрации.')
return
}
// Extract keyword from message
const messageText = ctx.message?.text || ''
const parts = messageText.split(' ')
if (parts.length < 2) {
await ctx.reply(
'Использование: /addkeyword <слово>\n\n' + 'Пример: /addkeyword строительство'
)
return
}
const keyword = parts.slice(1).join(' ').trim()
if (keyword.length === 0) {
await ctx.reply('Ключевое слово не может быть пустым.')
return
}
if (keyword.length > 255) {
await ctx.reply('Ключевое слово слишком длинное (максимум 255 символов).')
return
}
// Check if keyword already exists for this user
const existingKeyword = await Keyword.query()
.where('userId', user.id)
.where('keyword', keyword)
.first()
if (existingKeyword) {
await ctx.reply(`Ключевое слово "${keyword}" уже добавлено.`)
return
}
// Create keyword
const newKeyword = await Keyword.create({
userId: user.id,
keyword: keyword,
isActive: true,
})
await ctx.reply(`Ключевое слово "${keyword}" успешно добавлено (ID: ${newKeyword.id}).`)
logger.info(`Keyword added: "${keyword}" for user ${user.id} (chat_id: ${chatId})`)
}
/**
* Handle /listkeywords command - List user's keywords
*/
private async handleListKeywords(ctx: Context): Promise<void> {
if (!ctx.chat?.id) {
await ctx.reply('Не удалось определить chat ID.')
return
}
const chatId = String(ctx.chat.id)
const user = await User.findBy('telegramChatId', chatId)
if (!user) {
await ctx.reply('Вы не зарегистрированы. Используйте /start для регистрации.')
return
}
// Fetch user's keywords
const keywords = await Keyword.query().where('userId', user.id).where('isActive', true)
if (keywords.length === 0) {
await ctx.reply(
'У вас нет активных ключевых слов.\n\n' +
'Используйте /addkeyword для добавления ключевых слов.'
)
return
}
// Format keywords list
const keywordsList = keywords
.map((kw) => `${kw.id}. ${kw.keyword}`)
.join('\n')
await ctx.reply(`Ваши ключевые слова:\n\n${keywordsList}\n\nДля удаления используйте: /deletekeyword <id>`)
logger.info(`Listed ${keywords.length} keyword(s) for user ${user.id} (chat_id: ${chatId})`)
}
/**
* Handle /deletekeyword command - Delete keyword by ID
*/
private async handleDeleteKeyword(ctx: Context): Promise<void> {
if (!ctx.chat?.id) {
await ctx.reply('Не удалось определить chat ID.')
return
}
const chatId = String(ctx.chat.id)
const user = await User.findBy('telegramChatId', chatId)
if (!user) {
await ctx.reply('Вы не зарегистрированы. Используйте /start для регистрации.')
return
}
// Extract keyword ID from message
const messageText = ctx.message?.text || ''
const parts = messageText.split(' ')
if (parts.length < 2) {
await ctx.reply('Использование: /deletekeyword <id>\n\n' + 'Пример: /deletekeyword 5')
return
}
const keywordId = Number.parseInt(parts[1], 10)
if (Number.isNaN(keywordId)) {
await ctx.reply('Некорректный ID. Используйте числовое значение.')
return
}
// Find keyword
const keyword = await Keyword.query()
.where('id', keywordId)
.where('userId', user.id)
.first()
if (!keyword) {
await ctx.reply(`Ключевое слово с ID ${keywordId} не найдено.`)
return
}
// Soft delete - set isActive to false
keyword.isActive = false
await keyword.save()
await ctx.reply(`Ключевое слово "${keyword.keyword}" (ID: ${keywordId}) успешно удалено.`)
logger.info(
`Keyword deleted: "${keyword.keyword}" (ID: ${keywordId}) for user ${user.id} (chat_id: ${chatId})`
)
}
/**
* Handle /help command - Show help message
*/
private async handleHelp(ctx: Context): Promise<void> {
const helpMessage =
'🤖 Помощь по использованию бота\n\n' +
'Доступные команды:\n\n' +
'/start - Регистрация в системе\n' +
'/addkeyword <слово> - Добавить ключевое слово для отслеживания\n' +
'/listkeywords - Список ваших ключевых слов\n' +
'/deletekeyword <id> - Удалить ключевое слово\n' +
'/help - Показать эту справку\n\n' +
'Примеры использования:\n' +
'/addkeyword строительство\n' +
'/deletekeyword 5'
await ctx.reply(helpMessage)
}
/**
* Send notification to user about matched auction
*/
async sendNotification(
chatId: string,
auctionTitle: string,
auctionUrl: string,
keyword: string
): Promise<boolean> {
try {
const message =
`🔔 Новый аукцион по ключевому слову "${keyword}"\n\n` +
`📋 ${auctionTitle}\n\n` +
`🔗 ${auctionUrl}`
await this.bot.api.sendMessage(chatId, message, {
parse_mode: 'HTML',
})
logger.info(`Notification sent to chat ${chatId} for keyword "${keyword}"`)
return true
} catch (error) {
logger.error('Failed to send notification', {
error: error instanceof Error ? error.message : String(error),
chatId,
keyword,
})
return false
}
}
/**
* Start the bot (long polling)
*/
async start(): Promise<void> {
if (this.isRunning) {
logger.warn('Bot is already running')
return
}
try {
logger.info('Starting Telegram bot...')
this.isRunning = true
await this.bot.start()
logger.info('Telegram bot started successfully')
} catch (error) {
this.isRunning = false
logger.error('Failed to start Telegram bot', {
error: error instanceof Error ? error.message : String(error),
})
throw error
}
}
/**
* Stop the bot gracefully
*/
async stop(): Promise<void> {
if (!this.isRunning) {
logger.warn('Bot is not running')
return
}
try {
logger.info('Stopping Telegram bot...')
await this.bot.stop()
this.isRunning = false
logger.info('Telegram bot stopped successfully')
} catch (error) {
logger.error('Failed to stop Telegram bot', {
error: error instanceof Error ? error.message : String(error),
})
throw error
}
}
/**
* Check if bot is running
*/
getIsRunning(): boolean {
return this.isRunning
}
}

47
bin/console.ts Normal file
View File

@ -0,0 +1,47 @@
/*
|--------------------------------------------------------------------------
| Ace entry point
|--------------------------------------------------------------------------
|
| The "console.ts" file is the entrypoint for booting the AdonisJS
| command-line framework and executing commands.
|
| Commands do not boot the application, unless the currently running command
| has "options.startApp" flag set to true.
|
*/
import 'reflect-metadata'
import { Ignitor, prettyPrintError } from '@adonisjs/core'
/**
* URL to the application root. AdonisJS need it to resolve
* paths to file and directories for scaffolding commands
*/
const APP_ROOT = new URL('../', import.meta.url)
/**
* The importer is used to import files in context of the
* application.
*/
const IMPORTER = (filePath: string) => {
if (filePath.startsWith('./') || filePath.startsWith('../')) {
return import(new URL(filePath, APP_ROOT).href)
}
return import(filePath)
}
new Ignitor(APP_ROOT, { importer: IMPORTER })
.tap((app) => {
app.booting(async () => {
await import('#start/env')
})
app.listen('SIGTERM', () => app.terminate())
app.listenIf(app.managedByPm2, 'SIGINT', () => app.terminate())
})
.ace()
.handle(process.argv.splice(2))
.catch((error) => {
process.exitCode = 1
prettyPrintError(error)
})

45
bin/server.ts Normal file
View File

@ -0,0 +1,45 @@
/*
|--------------------------------------------------------------------------
| HTTP server entrypoint
|--------------------------------------------------------------------------
|
| The "server.ts" file is the entrypoint for starting the AdonisJS HTTP
| server. Either you can run this file directly or use the "serve"
| command to run this file and monitor file changes
|
*/
import 'reflect-metadata'
import { Ignitor, prettyPrintError } from '@adonisjs/core'
/**
* URL to the application root. AdonisJS need it to resolve
* paths to file and directories for scaffolding commands
*/
const APP_ROOT = new URL('../', import.meta.url)
/**
* The importer is used to import files in context of the
* application.
*/
const IMPORTER = (filePath: string) => {
if (filePath.startsWith('./') || filePath.startsWith('../')) {
return import(new URL(filePath, APP_ROOT).href)
}
return import(filePath)
}
new Ignitor(APP_ROOT, { importer: IMPORTER })
.tap((app) => {
app.booting(async () => {
await import('#start/env')
})
app.listen('SIGTERM', () => app.terminate())
app.listenIf(app.managedByPm2, 'SIGINT', () => app.terminate())
})
.httpServer()
.start()
.catch((error) => {
process.exitCode = 1
prettyPrintError(error)
})

62
bin/test.ts Normal file
View File

@ -0,0 +1,62 @@
/*
|--------------------------------------------------------------------------
| Test runner entrypoint
|--------------------------------------------------------------------------
|
| The "test.ts" file is the entrypoint for running tests using Japa.
|
| Either you can run this file directly or use the "test"
| command to run this file and monitor file changes.
|
*/
process.env.NODE_ENV = 'test'
import 'reflect-metadata'
import { Ignitor, prettyPrintError } from '@adonisjs/core'
import { configure, processCLIArgs, run } from '@japa/runner'
/**
* URL to the application root. AdonisJS need it to resolve
* paths to file and directories for scaffolding commands
*/
const APP_ROOT = new URL('../', import.meta.url)
/**
* The importer is used to import files in context of the
* application.
*/
const IMPORTER = (filePath: string) => {
if (filePath.startsWith('./') || filePath.startsWith('../')) {
return import(new URL(filePath, APP_ROOT).href)
}
return import(filePath)
}
new Ignitor(APP_ROOT, { importer: IMPORTER })
.tap((app) => {
app.booting(async () => {
await import('#start/env')
})
app.listen('SIGTERM', () => app.terminate())
app.listenIf(app.managedByPm2, 'SIGINT', () => app.terminate())
})
.testRunner()
.configure(async (app) => {
const { runnerHooks, ...config } = await import('../tests/bootstrap.js')
processCLIArgs(process.argv.splice(2))
configure({
...app.rcFile.tests,
...config,
...{
setup: runnerHooks.setup,
teardown: runnerHooks.teardown.concat([() => app.terminate()]),
},
})
})
.run(() => run())
.catch((error) => {
process.exitCode = 1
prettyPrintError(error)
})

207
commands/parse_auctions.ts Normal file
View File

@ -0,0 +1,207 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import { ScraperService } from '#services/scraper_service'
import { NotificationService } from '#services/notification_service'
import Auction from '#models/auction'
import ParseLog from '#models/parse_log'
import { DateTime } from 'luxon'
import logger from '@adonisjs/core/services/logger'
/**
* Command to parse auctions from icetrade.by
*
* Usage:
* node ace parse:auctions
* node ace parse:auctions --pages=5
* node ace parse:auctions --pages=3 --skip-notifications
*/
export default class ParseAuctions extends BaseCommand {
static commandName = 'parse:auctions'
static description = 'Scrape auctions from icetrade.by and store them in the database'
static options: CommandOptions = {
startApp: true,
allowUnknownFlags: false,
}
@flags.number({
description: 'Number of pages to scrape',
default: 1,
alias: 'p',
})
declare pages: number
@flags.boolean({
description: 'Skip sending notifications after parsing',
default: false,
})
declare skipNotifications: boolean
async run() {
const startTime = DateTime.now()
const scraper = new ScraperService()
const notificationService = new NotificationService()
// Create ParseLog entry with status 'running'
const parseLog = await ParseLog.create({
parseType: 'auction',
status: 'running',
itemsFound: 0,
errors: null,
startedAt: startTime,
completedAt: null,
})
this.logger.info(`Starting auction parsing (log ID: ${parseLog.id})`)
this.logger.info(`Scraping ${this.pages} page(s) from icetrade.by`)
let totalScraped = 0
let newAuctions = 0
let updatedAuctions = 0
let errors: string[] = []
try {
// Step 1: Scrape auctions
this.logger.info('Step 1: Scraping auctions...')
const auctionData = await scraper.scrapeAuctions(this.pages)
totalScraped = auctionData.length
this.logger.success(`Scraped ${totalScraped} auction(s)`)
if (totalScraped === 0) {
this.logger.warning('No auctions found to process')
await this.updateParseLog(parseLog, 'completed', 0, null)
return
}
// Step 2: Upsert auctions to database
this.logger.info('Step 2: Saving auctions to database...')
const processedAuctions: Auction[] = []
for (const data of auctionData) {
try {
// Check if auction exists by auctionNum
const existingAuction = await Auction.query()
.where('auctionNum', data.auctionNum)
.first()
let auction: Auction
if (existingAuction) {
// Update existing auction
existingAuction.title = data.title
existingAuction.description = data.description
existingAuction.organization = data.organization
existingAuction.status = data.status
existingAuction.deadline = data.deadline ? DateTime.fromISO(data.deadline) : null
existingAuction.url = data.link
existingAuction.rawData = data
await existingAuction.save()
auction = existingAuction
updatedAuctions++
logger.debug(`Updated auction: ${data.auctionNum}`)
} else {
// Create new auction
auction = await Auction.create({
auctionNum: data.auctionNum,
title: data.title,
description: data.description,
organization: data.organization,
status: data.status,
price: null, // Price not available in current schema
deadline: data.deadline ? DateTime.fromISO(data.deadline) : null,
url: data.link,
rawData: data,
})
newAuctions++
logger.debug(`Created auction: ${data.auctionNum}`)
// Step 3: Check for keyword matches and send notifications (for new auctions only)
if (!this.skipNotifications) {
try {
await notificationService.checkAndNotify(auction)
} catch (error) {
const errorMsg = `Notification check failed for auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
logger.error(errorMsg)
errors.push(errorMsg)
// Don't fail the entire process if notification fails
}
}
}
processedAuctions.push(auction)
} catch (error) {
const errorMsg = `Failed to save auction ${data.auctionNum}: ${error instanceof Error ? error.message : String(error)}`
logger.error(errorMsg)
errors.push(errorMsg)
}
}
this.logger.success(
`Saved ${processedAuctions.length} auction(s): ${newAuctions} new, ${updatedAuctions} updated`
)
if (this.skipNotifications) {
this.logger.info('Notifications skipped (--skip-notifications flag set)')
}
// Update ParseLog with success status
const status = errors.length > 0 ? 'completed_with_errors' : 'completed'
await this.updateParseLog(
parseLog,
status,
processedAuctions.length,
errors.length > 0 ? errors.join('\n') : null
)
// Summary
const duration = DateTime.now().diff(startTime).toFormat("m 'min' s 'sec'")
this.logger.info('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━')
this.logger.info('Parsing Summary:')
this.logger.info(` Duration: ${duration}`)
this.logger.info(` Total scraped: ${totalScraped}`)
this.logger.info(` New auctions: ${newAuctions}`)
this.logger.info(` Updated auctions: ${updatedAuctions}`)
if (errors.length > 0) {
this.logger.warning(` Errors: ${errors.length}`)
}
this.logger.info('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━')
if (errors.length > 0) {
this.logger.warning('Parsing completed with errors')
this.exitCode = 1
} else {
this.logger.success('Parsing completed successfully')
}
} catch (error) {
// Handle catastrophic failure
const errorMsg = error instanceof Error ? error.message : String(error)
logger.error('Fatal error during auction parsing', { error: errorMsg })
await this.updateParseLog(parseLog, 'failed', 0, errorMsg)
this.logger.error('Parsing failed: ' + errorMsg)
this.exitCode = 1
}
}
/**
* Update ParseLog entry with final status
*/
private async updateParseLog(
parseLog: ParseLog,
status: string,
itemsFound: number,
errors: string | null
): Promise<void> {
parseLog.status = status
parseLog.itemsFound = itemsFound
parseLog.errors = errors
parseLog.completedAt = DateTime.now()
await parseLog.save()
logger.info(`Updated ParseLog ${parseLog.id}: status=${status}, items=${itemsFound}`)
}
}

View File

@ -0,0 +1,86 @@
import { BaseCommand } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import { TelegramService } from '#services/telegram_service'
import logger from '@adonisjs/core/services/logger'
/**
* Command to start Telegram bot
*
* Usage:
* node ace telegram:start
*
* This command runs as a long-lived process and handles graceful shutdown.
*/
export default class StartTelegramBot extends BaseCommand {
static commandName = 'telegram:start'
static description = 'Start the Telegram bot for auction notifications'
static options: CommandOptions = {
startApp: true,
staysAlive: true,
}
private telegramService: TelegramService | null = null
async run() {
try {
this.logger.info('Initializing Telegram bot service...')
// Get singleton instance
this.telegramService = TelegramService.getInstance()
// Setup graceful shutdown handlers
this.setupShutdownHandlers()
// Start the bot
this.logger.info('Starting Telegram bot...')
await this.telegramService.start()
this.logger.success('Telegram bot is running! Press Ctrl+C to stop.')
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error)
this.logger.error(`Failed to start Telegram bot: ${errorMsg}`)
logger.error('Telegram bot startup failed', { error: errorMsg })
this.exitCode = 1
await this.terminate()
}
}
/**
* Setup handlers for graceful shutdown
*/
private setupShutdownHandlers(): void {
const shutdown = async (signal: string) => {
this.logger.info(`Received ${signal}, shutting down gracefully...`)
if (this.telegramService) {
try {
await this.telegramService.stop()
this.logger.success('Telegram bot stopped successfully')
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error)
this.logger.error(`Error during shutdown: ${errorMsg}`)
logger.error('Telegram bot shutdown error', { error: errorMsg })
}
}
await this.terminate()
}
// Handle SIGINT (Ctrl+C)
process.on('SIGINT', () => shutdown('SIGINT'))
// Handle SIGTERM (kill command)
process.on('SIGTERM', () => shutdown('SIGTERM'))
}
/**
* Command termination handler
*/
async completed() {
if (this.telegramService && this.telegramService.getIsRunning()) {
this.logger.info('Stopping Telegram bot...')
await this.telegramService.stop()
}
}
}

147
commands/test_connection.ts Normal file
View File

@ -0,0 +1,147 @@
import { BaseCommand } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import logger from '@adonisjs/core/services/logger'
/**
* Test network connectivity to icetrade.by
*
* Usage:
* node ace test:connection
*/
export default class TestConnection extends BaseCommand {
static commandName = 'test:connection'
static description = 'Test network connectivity to icetrade.by'
static options: CommandOptions = {
startApp: true,
}
async run() {
const testUrl = 'https://icetrade.by'
this.logger.info('Testing network connectivity...')
this.logger.info(`Target URL: ${testUrl}`)
try {
// Test 1: Basic DNS resolution
this.logger.info('Test 1: DNS resolution')
try {
const dns = await import('node:dns/promises')
const addresses = await dns.resolve4('icetrade.by')
this.logger.success(`DNS resolved: ${addresses.join(', ')}`)
} catch (error) {
this.logger.error(
`DNS resolution failed: ${error instanceof Error ? error.message : String(error)}`
)
throw error
}
// Test 2: Basic HTTP connection
this.logger.info('Test 2: HTTP connection')
const startTime = Date.now()
const controller = new AbortController()
const timeoutId = setTimeout(() => controller.abort(), 30000)
try {
const response = await fetch(testUrl, {
method: 'HEAD',
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
},
signal: controller.signal,
})
clearTimeout(timeoutId)
const duration = Date.now() - startTime
this.logger.success(`HTTP response: ${response.status} ${response.statusText}`)
this.logger.info(`Response time: ${duration}ms`)
this.logger.info(`Content-Type: ${response.headers.get('content-type')}`)
this.logger.info(`Server: ${response.headers.get('server')}`)
} catch (error) {
clearTimeout(timeoutId)
const err = error instanceof Error ? error : new Error(String(error))
this.logger.error(`HTTP connection failed: ${err.message}`)
// Log detailed error info
logger.error('Connection test failed', {
name: err.name,
message: err.message,
stack: err.stack,
cause: err.cause,
})
throw error
}
// Test 3: Full page fetch
this.logger.info('Test 3: Full page fetch')
try {
const fullResponse = await fetch(testUrl, {
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
Accept: 'text/html',
},
})
const text = await fullResponse.text()
this.logger.success(`Received ${text.length} bytes`)
this.logger.info(`First 200 chars: ${text.substring(0, 200).replace(/\s+/g, ' ')}`)
} catch (error) {
this.logger.error(
`Full fetch failed: ${error instanceof Error ? error.message : String(error)}`
)
throw error
}
// Test 4: Actual auction page
this.logger.info('Test 4: Auction list page')
const auctionUrl = 'https://icetrade.by/trades/index?onPage=100&p=1'
try {
const auctionResponse = await fetch(auctionUrl, {
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
Accept: 'text/html',
'Accept-Language': 'ru-RU,ru;q=0.9',
},
})
if (!auctionResponse.ok) {
throw new Error(
`HTTP ${auctionResponse.status}: ${auctionResponse.statusText}`
)
}
const html = await auctionResponse.text()
this.logger.success(`Auction page received: ${html.length} bytes`)
// Check for auction table
if (html.includes('table.auctions') || html.includes('class="auctions')) {
this.logger.success('Auction table found in HTML')
} else {
this.logger.warning('Auction table not found in HTML (page structure may have changed)')
}
} catch (error) {
this.logger.error(
`Auction page fetch failed: ${error instanceof Error ? error.message : String(error)}`
)
throw error
}
this.logger.success('All connectivity tests passed!')
} catch (error) {
this.logger.error('Connectivity test failed')
this.exitCode = 1
// Print environment info
this.logger.info('Environment info:')
this.logger.info(` NODE_ENV: ${process.env.NODE_ENV}`)
this.logger.info(` Platform: ${process.platform}`)
this.logger.info(` Node version: ${process.version}`)
}
}
}

40
config/app.ts Normal file
View File

@ -0,0 +1,40 @@
import env from '#start/env'
import app from '@adonisjs/core/services/app'
import { Secret } from '@adonisjs/core/helpers'
import { defineConfig } from '@adonisjs/core/http'
/**
* The app key is used for encrypting cookies, generating signed URLs,
* and by the "encryption" module.
*
* The encryption module will fail to decrypt data if the key is lost or
* changed. Therefore it is recommended to keep the app key secure.
*/
export const appKey = new Secret(env.get('APP_KEY'))
/**
* The configuration settings used by the HTTP server
*/
export const http = defineConfig({
generateRequestId: true,
allowMethodSpoofing: false,
/**
* Enabling async local storage will let you access HTTP context
* from anywhere inside your application.
*/
useAsyncLocalStorage: false,
/**
* Manage cookies configuration. The settings for the session id cookie are
* defined inside the "config/session.ts" file.
*/
cookie: {
domain: '',
path: '/',
maxAge: '2h',
httpOnly: true,
secure: app.inProduction,
sameSite: 'lax',
},
})

28
config/auth.ts Normal file
View File

@ -0,0 +1,28 @@
import { defineConfig } from '@adonisjs/auth'
import { sessionGuard, sessionUserProvider } from '@adonisjs/auth/session'
import type { InferAuthenticators, InferAuthEvents, Authenticators } from '@adonisjs/auth/types'
const authConfig = defineConfig({
default: 'web',
guards: {
web: sessionGuard({
useRememberMeTokens: false,
provider: sessionUserProvider({
model: () => import('#models/user')
}),
}),
},
})
export default authConfig
/**
* Inferring types from the configured auth
* guards.
*/
declare module '@adonisjs/auth/types' {
export interface Authenticators extends InferAuthenticators<typeof authConfig> {}
}
declare module '@adonisjs/core/types' {
interface EventsList extends InferAuthEvents<Authenticators> {}
}

55
config/bodyparser.ts Normal file
View File

@ -0,0 +1,55 @@
import { defineConfig } from '@adonisjs/core/bodyparser'
const bodyParserConfig = defineConfig({
/**
* The bodyparser middleware will parse the request body
* for the following HTTP methods.
*/
allowedMethods: ['POST', 'PUT', 'PATCH', 'DELETE'],
/**
* Config for the "application/x-www-form-urlencoded"
* content-type parser
*/
form: {
convertEmptyStringsToNull: true,
types: ['application/x-www-form-urlencoded'],
},
/**
* Config for the JSON parser
*/
json: {
convertEmptyStringsToNull: true,
types: [
'application/json',
'application/json-patch+json',
'application/vnd.api+json',
'application/csp-report',
],
},
/**
* Config for the "multipart/form-data" content-type parser.
* File uploads are handled by the multipart parser.
*/
multipart: {
/**
* Enabling auto process allows bodyparser middleware to
* move all uploaded files inside the tmp folder of your
* operating system
*/
autoProcess: true,
convertEmptyStringsToNull: true,
processManually: [],
/**
* Maximum limit of data to parse including all files
* and fields
*/
limit: '20mb',
types: ['multipart/form-data'],
},
})
export default bodyParserConfig

24
config/database.ts Normal file
View File

@ -0,0 +1,24 @@
import env from '#start/env'
import { defineConfig } from '@adonisjs/lucid'
const dbConfig = defineConfig({
connection: 'postgres',
connections: {
postgres: {
client: 'pg',
connection: {
host: env.get('DB_HOST'),
port: env.get('DB_PORT'),
user: env.get('DB_USER'),
password: env.get('DB_PASSWORD'),
database: env.get('DB_DATABASE'),
},
migrations: {
naturalSort: true,
paths: ['database/migrations'],
},
},
},
})
export default dbConfig

24
config/hash.ts Normal file
View File

@ -0,0 +1,24 @@
import { defineConfig, drivers } from '@adonisjs/core/hash'
const hashConfig = defineConfig({
default: 'scrypt',
list: {
scrypt: drivers.scrypt({
cost: 16384,
blockSize: 8,
parallelization: 1,
maxMemory: 33554432,
}),
},
})
export default hashConfig
/**
* Inferring types for the list of hashers you have configured
* in your application.
*/
declare module '@adonisjs/core/types' {
export interface HashersList extends InferHashers<typeof hashConfig> {}
}

40
config/logger.ts Normal file
View File

@ -0,0 +1,40 @@
import env from '#start/env'
import app from '@adonisjs/core/services/app'
import { defineConfig, targets } from '@adonisjs/core/logger'
const loggerConfig = defineConfig({
default: 'app',
/**
* The loggers object can be used to define multiple loggers.
* By default, we configure only one logger (named "app").
*/
loggers: {
app: {
enabled: true,
name: env.get('APP_NAME'),
level: env.get('LOG_LEVEL'),
transport: {
targets: targets()
.pushIf(!app.inProduction, targets.pretty({ colorize: true }))
.pushIf(app.inProduction, targets.file({ destination: 1 }))
.toArray(),
},
// Redact sensitive fields from logs
redact: {
paths: ['password', 'APP_KEY', 'TELEGRAM_BOT_TOKEN'],
censor: '***REDACTED***',
},
},
},
})
export default loggerConfig
/**
* Inferring types for the list of loggers you have configured
* in your application.
*/
declare module '@adonisjs/core/types' {
export interface LoggersList extends InferLoggers<typeof loggerConfig> {}
}

48
config/session.ts Normal file
View File

@ -0,0 +1,48 @@
import env from '#start/env'
import app from '@adonisjs/core/services/app'
import { defineConfig, stores } from '@adonisjs/session'
const sessionConfig = defineConfig({
enabled: true,
cookieName: 'adonis-session',
/**
* When set to true, the session id cookie will be deleted
* once the user closes the browser.
*/
clearWithBrowser: false,
/**
* Define how long to keep the session data alive without
* any activity.
*/
age: '2h',
/**
* Configuration for session cookie and the
* cookie store
*/
cookie: {
path: '/',
httpOnly: true,
secure: app.inProduction,
sameSite: 'lax',
},
/**
* The store to use. Make sure to validate the environment
* variable in order to infer the store name without any
* errors.
*/
store: env.get('SESSION_DRIVER'),
/**
* List of configured stores. Refer documentation to see
* list of available stores and their config.
*/
stores: {
cookie: stores.cookie(),
},
})
export default sessionConfig

51
config/shield.ts Normal file
View File

@ -0,0 +1,51 @@
import { defineConfig } from '@adonisjs/shield'
const shieldConfig = defineConfig({
/**
* Configure CSP policies for your app. Refer documentation
* to learn more
*/
csp: {
enabled: false,
directives: {},
reportOnly: false,
},
/**
* Configure CSRF protection options. Refer documentation
* to learn more
*/
csrf: {
enabled: true,
exceptRoutes: [],
enableXsrfCookie: false,
methods: ['POST', 'PUT', 'PATCH', 'DELETE'],
},
/**
* Control how your website should be embedded inside
* iFrames
*/
xFrame: {
enabled: true,
action: 'DENY',
},
/**
* Force browser to always use HTTPS
*/
hsts: {
enabled: true,
maxAge: '180 days',
},
/**
* Disable browsers from sniffing the content type of a
* response and always rely on the "content-type" header.
*/
contentTypeSniffing: {
enabled: true,
},
})
export default shieldConfig

17
config/static.ts Normal file
View File

@ -0,0 +1,17 @@
import { defineConfig } from '@adonisjs/static'
/**
* Configuration options to tweak the static files middleware.
* The complete set of options are documented on the
* official documentation website.
*
* https://docs.adonisjs.com/guides/basics/static-file-server
*/
const staticServerConfig = defineConfig({
enabled: true,
etag: true,
lastModified: true,
dotFiles: 'ignore',
})
export default staticServerConfig

28
config/vite.ts Normal file
View File

@ -0,0 +1,28 @@
import { defineConfig } from '@adonisjs/vite'
const viteBackendConfig = defineConfig({
/**
* The output of vite will be written inside this
* directory. The path should be relative from
* the application root.
*/
buildDirectory: 'public/assets',
/**
* The path to the manifest file generated by the
* "vite build" command.
*/
manifestFile: 'public/assets/.vite/manifest.json',
/**
* Feel free to change the value of the "assetsUrl" to
* point to a CDN in production.
*/
assetsUrl: '/assets',
scriptAttributes: {
defer: true,
},
})
export default viteBackendConfig

View File

@ -0,0 +1,26 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'auctions'
async up() {
this.schema.createTable(this.tableName, (table) => {
table.increments('id').primary()
table.string('auction_num', 255).notNullable().unique()
table.string('title', 500).notNullable()
table.text('description').nullable()
table.string('organization', 500).nullable()
table.string('status', 100).nullable()
table.decimal('price', 15, 2).nullable()
table.timestamp('deadline', { useTz: true }).nullable()
table.string('url', 1000).nullable()
table.json('raw_data').nullable()
table.timestamp('created_at', { useTz: true }).notNullable()
table.timestamp('updated_at', { useTz: true }).notNullable()
})
}
async down() {
this.schema.dropTable(this.tableName)
}
}

View File

@ -0,0 +1,19 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'keywords'
async up() {
this.schema.createTable(this.tableName, (table) => {
table.increments('id').primary()
table.string('keyword', 255).notNullable().unique()
table.boolean('is_active').notNullable().defaultTo(true)
table.timestamp('created_at', { useTz: true }).notNullable()
table.timestamp('updated_at', { useTz: true }).notNullable()
})
}
async down() {
this.schema.dropTable(this.tableName)
}
}

View File

@ -0,0 +1,20 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'users'
async up() {
this.schema.createTable(this.tableName, (table) => {
table.increments('id').primary()
table.string('email', 255).notNullable().unique()
table.string('password', 180).notNullable()
table.string('telegram_chat_id', 255).nullable().unique()
table.timestamp('created_at', { useTz: true }).notNullable()
table.timestamp('updated_at', { useTz: true }).notNullable()
})
}
async down() {
this.schema.dropTable(this.tableName)
}
}

View File

@ -0,0 +1,20 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'notifications'
async up() {
this.schema.createTable(this.tableName, (table) => {
table.increments('id').primary()
table.integer('auction_id').unsigned().notNullable().references('id').inTable('auctions').onDelete('CASCADE')
table.string('notification_type', 100).notNullable()
table.timestamp('sent_at', { useTz: true }).notNullable()
table.timestamp('created_at', { useTz: true }).notNullable()
table.timestamp('updated_at', { useTz: true }).notNullable()
})
}
async down() {
this.schema.dropTable(this.tableName)
}
}

View File

@ -0,0 +1,23 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'parse_logs'
async up() {
this.schema.createTable(this.tableName, (table) => {
table.increments('id').primary()
table.string('parse_type', 100).notNullable()
table.string('status', 50).notNullable()
table.integer('items_found').notNullable().defaultTo(0)
table.text('errors').nullable()
table.timestamp('started_at', { useTz: true }).notNullable()
table.timestamp('completed_at', { useTz: true }).nullable()
table.timestamp('created_at', { useTz: true }).notNullable()
table.timestamp('updated_at', { useTz: true }).notNullable()
})
}
async down() {
this.schema.dropTable(this.tableName)
}
}

View File

@ -0,0 +1,29 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'keywords'
async up() {
this.schema.alterTable(this.tableName, (table) => {
table.integer('user_id').unsigned().references('id').inTable('users').onDelete('CASCADE').nullable()
table.dropUnique(['keyword'])
})
// Add composite unique constraint for user_id + keyword
this.schema.alterTable(this.tableName, (table) => {
table.unique(['user_id', 'keyword'])
})
}
async down() {
this.schema.alterTable(this.tableName, (table) => {
table.dropUnique(['user_id', 'keyword'])
})
this.schema.alterTable(this.tableName, (table) => {
table.dropForeign(['user_id'])
table.dropColumn('user_id')
table.unique(['keyword'])
})
}
}

View File

@ -0,0 +1,55 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'notifications'
async up() {
this.schema.alterTable(this.tableName, (table) => {
// Add keyword_id foreign key
table
.integer('keyword_id')
.unsigned()
.notNullable()
.references('id')
.inTable('keywords')
.onDelete('CASCADE')
// Add status enum with default 'pending'
table.enum('status', ['pending', 'sent', 'failed']).notNullable().defaultTo('pending')
// Add error_message for failed notifications
table.text('error_message').nullable()
// Alter sent_at to be nullable (was NOT NULL before)
table.timestamp('sent_at', { useTz: true }).nullable().alter()
// Remove notification_type column as it's replaced by status
table.dropColumn('notification_type')
// Add composite index for auction_id + keyword_id to prevent duplicates
table.index(['auction_id', 'keyword_id'])
// Add index on status for filtering
table.index('status')
})
}
async down() {
this.schema.alterTable(this.tableName, (table) => {
// Drop indexes
table.dropIndex(['auction_id', 'keyword_id'])
table.dropIndex('status')
// Restore notification_type column
table.string('notification_type', 100).notNullable()
// Restore sent_at as NOT NULL
table.timestamp('sent_at', { useTz: true }).notNullable().alter()
// Remove new columns
table.dropColumn('error_message')
table.dropColumn('status')
table.dropColumn('keyword_id')
})
}
}

View File

@ -0,0 +1,18 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'keywords'
async up() {
this.schema.alterTable(this.tableName, (table) => {
// Add case_sensitive flag with default false
table.boolean('case_sensitive').notNullable().defaultTo(false)
})
}
async down() {
this.schema.alterTable(this.tableName, (table) => {
table.dropColumn('case_sensitive')
})
}
}

147
docker-compose.yml Normal file
View File

@ -0,0 +1,147 @@
services:
# PostgreSQL database
postgres:
image: postgres:15-alpine
container_name: parser_postgres
restart: unless-stopped
environment:
POSTGRES_USER: ${DB_USER:-postgres}
POSTGRES_PASSWORD: ${DB_PASSWORD:-postgres}
POSTGRES_DB: ${DB_DATABASE:-parser_zakupok}
ports:
- "${DB_PORT:-5432}:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres}"]
interval: 10s
timeout: 5s
retries: 5
networks:
- parser_network
# Redis cache (for future use with queues/caching)
redis:
image: redis:7-alpine
container_name: parser_redis
restart: unless-stopped
ports:
- "6379:6379"
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 3s
retries: 5
networks:
- parser_network
# Main AdonisJS application (HTTP API server)
app:
build:
context: .
dockerfile: Dockerfile
target: production
container_name: parser_app
restart: unless-stopped
ports:
- "${PORT:-3333}:3333"
environment:
TZ: ${TZ:-UTC}
PORT: 3333
HOST: 0.0.0.0
LOG_LEVEL: ${LOG_LEVEL:-info}
APP_KEY: ${APP_KEY}
NODE_ENV: production
SESSION_DRIVER: ${SESSION_DRIVER:-cookie}
DB_HOST: postgres
DB_PORT: 5432
DB_USER: ${DB_USER:-postgres}
DB_PASSWORD: ${DB_PASSWORD:-postgres}
DB_DATABASE: ${DB_DATABASE:-parser_zakupok}
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN}
ICETRADE_BASE_URL: ${ICETRADE_BASE_URL:-https://icetrade.by}
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
networks:
- parser_network
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3333/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
# Scheduler service (runs scheduled auction parsing every 6 hours)
scheduler:
build:
context: .
dockerfile: Dockerfile
target: production
container_name: parser_scheduler
restart: unless-stopped
command: ["node", "ace", "scheduler:work"]
# Use host network to bypass Docker networking issues (Windows)
network_mode: host
environment:
TZ: ${TZ:-UTC}
LOG_LEVEL: ${LOG_LEVEL:-info}
APP_KEY: ${APP_KEY}
NODE_ENV: production
SESSION_DRIVER: ${SESSION_DRIVER:-cookie}
# Connect to postgres via localhost when using host network
DB_HOST: ${DB_HOST:-localhost}
DB_PORT: ${DB_PORT:-5433}
DB_USER: ${DB_USER:-postgres}
DB_PASSWORD: ${DB_PASSWORD:-postgres}
DB_DATABASE: ${DB_DATABASE:-parser_zakupok}
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN}
ICETRADE_BASE_URL: ${ICETRADE_BASE_URL:-https://icetrade.by}
# Uncomment if using proxy:
# HTTP_PROXY: ${HTTP_PROXY}
# HTTPS_PROXY: ${HTTPS_PROXY}
# NO_PROXY: localhost,127.0.0.1
depends_on:
postgres:
condition: service_healthy
# Telegram bot service (runs continuously)
telegram:
build:
context: .
dockerfile: Dockerfile
target: production
container_name: parser_telegram
restart: unless-stopped
command: ["node", "ace", "telegram:start"]
environment:
TZ: ${TZ:-UTC}
LOG_LEVEL: ${LOG_LEVEL:-info}
APP_KEY: ${APP_KEY}
NODE_ENV: production
SESSION_DRIVER: ${SESSION_DRIVER:-cookie}
DB_HOST: postgres
DB_PORT: 5432
DB_USER: ${DB_USER:-postgres}
DB_PASSWORD: ${DB_PASSWORD:-postgres}
DB_DATABASE: ${DB_DATABASE:-parser_zakupok}
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN}
depends_on:
postgres:
condition: service_healthy
networks:
- parser_network
volumes:
postgres_data:
driver: local
redis_data:
driver: local
networks:
parser_network:
driver: bridge

2
eslint.config.js Normal file
View File

@ -0,0 +1,2 @@
import { configApp } from '@adonisjs/eslint-config'
export default configApp()

9048
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

79
package.json Normal file
View File

@ -0,0 +1,79 @@
{
"name": "parser-test",
"version": "0.0.0",
"private": true,
"type": "module",
"license": "UNLICENSED",
"scripts": {
"start": "node bin/server.js",
"build": "node ace build",
"dev": "node ace serve --hmr",
"test": "node ace test",
"lint": "eslint .",
"format": "prettier --write .",
"typecheck": "tsc --noEmit"
},
"imports": {
"#controllers/*": "./app/controllers/*.js",
"#exceptions/*": "./app/exceptions/*.js",
"#models/*": "./app/models/*.js",
"#mails/*": "./app/mails/*.js",
"#services/*": "./app/services/*.js",
"#listeners/*": "./app/listeners/*.js",
"#events/*": "./app/events/*.js",
"#middleware/*": "./app/middleware/*.js",
"#validators/*": "./app/validators/*.js",
"#providers/*": "./providers/*.js",
"#policies/*": "./app/policies/*.js",
"#abilities/*": "./app/abilities/*.js",
"#database/*": "./database/*.js",
"#tests/*": "./tests/*.js",
"#start/*": "./start/*.js",
"#config/*": "./config/*.js"
},
"devDependencies": {
"@adonisjs/assembler": "^7.8.2",
"@adonisjs/eslint-config": "^2.0.0",
"@adonisjs/prettier-config": "^1.4.4",
"@adonisjs/tsconfig": "^1.4.0",
"@japa/assert": "^4.0.1",
"@japa/plugin-adonisjs": "^4.0.0",
"@japa/runner": "^4.2.0",
"@swc/core": "1.11.24",
"@types/luxon": "^3.7.1",
"@types/node": "^22.15.18",
"@types/pg": "^8.15.5",
"eslint": "^9.26.0",
"hot-hook": "^0.4.0",
"pino-pretty": "^13.0.0",
"prettier": "^3.5.3",
"ts-node-maintained": "^10.9.5",
"typescript": "~5.8",
"vite": "^6.3.5"
},
"dependencies": {
"@adonisjs/auth": "^9.4.0",
"@adonisjs/core": "^6.18.0",
"@adonisjs/lucid": "^21.6.1",
"@adonisjs/session": "^7.5.1",
"@adonisjs/shield": "^8.2.0",
"@adonisjs/static": "^1.1.1",
"@adonisjs/vite": "^4.0.0",
"@vinejs/vine": "^3.0.1",
"adonisjs-scheduler": "^2.5.0",
"cheerio": "^1.1.2",
"edge.js": "^6.2.1",
"grammy": "^1.38.3",
"luxon": "^3.7.2",
"pg": "^8.16.3",
"reflect-metadata": "^0.2.2",
"zod": "^4.1.12"
},
"hotHook": {
"boundaries": [
"./app/controllers/**/*.ts",
"./app/middleware/*.ts"
]
},
"prettier": "@adonisjs/prettier-config"
}

10
resources/css/app.css Normal file
View File

@ -0,0 +1,10 @@
* {
margin: 0;
padding: 0;
}
html,
body {
height: 100%;
width: 100%;
}

1
resources/js/app.js Normal file
View File

@ -0,0 +1 @@
console.log('Hello World')

View File

@ -0,0 +1,175 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Auctions - Last 3 Days</title>
<script src="https://cdn.tailwindcss.com"></script>
<style>
body {
font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
}
.table-container {
overflow-x: auto;
}
</style>
</head>
<body class="bg-gray-50 min-h-screen">
<div class="container mx-auto px-4 py-8">
<div class="bg-white rounded-lg shadow-md p-6">
<div class="mb-6">
<h1 class="text-3xl font-bold text-gray-800">Auctions - Last 3 Days</h1>
<p class="text-gray-600 mt-2">Total auctions: {{ auctions.length }}</p>
<p class="text-gray-500 text-sm">From: {{ fromDate }}</p>
</div>
{{-- Flash messages --}}
@if(flashMessages.has('success'))
<div class="mb-6 bg-green-50 border border-green-200 text-green-800 px-4 py-3 rounded-lg" role="alert">
<div class="flex items-center">
<svg class="w-5 h-5 mr-2" fill="currentColor" viewBox="0 0 20 20">
<path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z" clip-rule="evenodd"/>
</svg>
<span>{{ flashMessages.get('success') }}</span>
</div>
</div>
@end
@if(flashMessages.has('error'))
<div class="mb-6 bg-red-50 border border-red-200 text-red-800 px-4 py-3 rounded-lg" role="alert">
<div class="flex items-center">
<svg class="w-5 h-5 mr-2" fill="currentColor" viewBox="0 0 20 20">
<path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clip-rule="evenodd"/>
</svg>
<span>{{ flashMessages.get('error') }}</span>
</div>
</div>
@end
{{-- Parsing form --}}
<div class="mb-8 bg-blue-50 border border-blue-200 rounded-lg p-6">
<h2 class="text-xl font-semibold text-gray-800 mb-4">Parse New Auctions</h2>
<form id="parseForm" method="POST" action="/trigger-parse" class="space-y-4">
{{ csrfField() }}
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
<div>
<label for="pages" class="block text-sm font-medium text-gray-700 mb-2">
Number of pages to parse
</label>
<input
type="number"
id="pages"
name="pages"
min="1"
max="10"
value="1"
required
class="w-full px-4 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-blue-500 focus:border-transparent"
/>
<p class="mt-1 text-xs text-gray-500">Min: 1, Max: 10 pages</p>
</div>
<div class="flex items-center">
<div class="flex items-center h-full">
<input
type="checkbox"
id="notifySubscribers"
name="notifySubscribers"
checked
class="w-5 h-5 text-blue-600 border-gray-300 rounded focus:ring-2 focus:ring-blue-500"
/>
<label for="notifySubscribers" class="ml-3 text-sm font-medium text-gray-700">
Notify subscribers about new auctions
</label>
</div>
</div>
</div>
<div class="flex items-center gap-4">
<button
type="submit"
id="submitBtn"
class="px-6 py-2 bg-blue-600 hover:bg-blue-700 text-white font-medium rounded-lg transition-colors duration-150 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2"
>
Start Parsing
</button>
<div id="loadingIndicator" class="hidden flex items-center text-blue-600">
<svg class="animate-spin h-5 w-5 mr-2" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
<span class="font-medium">Parsing in progress...</span>
</div>
</div>
</form>
</div>
<script>
document.getElementById('parseForm').addEventListener('submit', function() {
document.getElementById('submitBtn').disabled = true;
document.getElementById('loadingIndicator').classList.remove('hidden');
});
</script>
@if(auctions.length === 0)
<div class="text-center py-12">
<p class="text-gray-500 text-lg">No auctions found in the last 3 days.</p>
</div>
@else
<div class="table-container">
<table class="min-w-full divide-y divide-gray-200">
<thead class="bg-gray-100">
<tr>
<th class="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">
Auction Number
</th>
<th class="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">
Title
</th>
<th class="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">
Description
</th>
<th class="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">
Start Price
</th>
<th class="px-6 py-3 text-left text-xs font-medium text-gray-700 uppercase tracking-wider">
Created Date
</th>
</tr>
</thead>
<tbody class="bg-white divide-y divide-gray-200">
@each(auction in auctions)
<tr class="hover:bg-gray-50 transition-colors duration-150">
<td class="px-6 py-4 whitespace-nowrap text-sm font-medium text-blue-600">
{{ auction.auctionNum }}
</td>
<td class="px-6 py-4 text-sm text-gray-900">
<div class="max-w-md">
{{ auction.title || 'N/A' }}
</div>
</td>
<td class="px-6 py-4 text-sm text-gray-700">
<div class="max-w-lg line-clamp-3">
{{ auction.description ? (auction.description.length > 150 ? auction.description.substring(0, 150) + '...' : auction.description) : 'N/A' }}
</div>
</td>
<td class="px-6 py-4 whitespace-nowrap text-sm text-gray-900 font-semibold">
@if(auction.startPrice)
{{ new Intl.NumberFormat('ru-BY', { style: 'currency', currency: 'BYN' }).format(auction.startPrice) }}
@else
<span class="text-gray-400">N/A</span>
@end
</td>
<td class="px-6 py-4 whitespace-nowrap text-sm text-gray-600">
{{ auction.createdAt.toFormat('dd.MM.yyyy HH:mm') }}
</td>
</tr>
@end
</tbody>
</table>
</div>
@end
</div>
</div>
</body>
</html>

View File

@ -0,0 +1,8 @@
<h1>
404 - Page not found
</h1>
<p>
This template is rendered by the
<a href="http://docs.adonisjs.com/guides/exception-handling#status-pages">status pages feature</a>
of the global exception handler.
</p>

View File

@ -0,0 +1,8 @@
<h1>
{{ error.code }} - Server error
</h1>
<p>
This template is rendered by the
<a href="http://docs.adonisjs.com/guides/exception-handling#status-pages">status pages feature</a>
of the global exception handler.
</p>

View File

@ -0,0 +1,408 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>
AdonisJS - A fully featured web framework for Node.js
</title>
<link rel="preconnect" href="https://fonts.bunny.net" />
<link
href="https://fonts.bunny.net/css?family=instrument-sans:400,400i,500,500i,600,600i,700,700i"
rel="stylesheet"
/>
<style>
:root {
--sand-1: #fdfdfc;
--sand-2: #f9f9f8;
--sand-3: #f1f0ef;
--sand-4: #e9e8e6;
--sand-5: #e2e1de;
--sand-6: #dad9d6;
--sand-7: #cfceca;
--sand-8: #bcbbb5;
--sand-9: #8d8d86;
--sand-10: #82827c;
--sand-11: #63635e;
--sand-12: #21201c;
}
</style>
<script src="https://cdn.tailwindcss.com"></script>
<script>
tailwind.config = {
theme: {
extend: {
fontFamily: {
sans: [ "Instrument Sans", "sans-serif" ]
},
colors: {
primary: {
DEFAULT: "#5A45FF",
lighter: "#a599ff"
},
sand: {
1: "var(--sand-1)",
2: "var(--sand-2)",
3: "var(--sand-3)",
4: "var(--sand-4)",
5: "var(--sand-5)",
6: "var(--sand-6)",
7: "var(--sand-7)",
8: "var(--sand-8)",
9: "var(--sand-9)",
10: "var(--sand-10)",
11: "var(--sand-11)",
12: "var(--sand-12)"
}
}
}
}
};
</script>
@vite(['resources/css/app.css', 'resources/js/app.js'])
@stack('dumper')
</head>
<body class="min-h-screen w-screen font-sans">
<div
class="fixed xl:absolute left-8 right-8 top-0 bottom-0 xl:inset-0 max-w-screen-xl mx-auto before:content-[''] before:[background:repeating-linear-gradient(0deg,var(--sand-5)_0_4px,transparent_0_8px)] before:absolute before:top-0 before:left-0 before:h-full before:w-px after:content-[''] after:[background:repeating-linear-gradient(0deg,var(--sand-5)_0_4px,transparent_0_8px)] after:absolute after:top-0 after:right-0 after:h-full after:w-px"
>
</div>
<div class="pt-4 h-full flex flex-col">
{{-- Header --}}
<div class="grow pb-4 bg-gradient-to-b from-sand-1 to-sand-2 flex justify-center items-center">
<a href="https://adonisjs.com" target="_blank" class="isolate">
<svg class="w-16 h-16 fill-primary" viewBox="0 0 33 33">
<path
fill-rule="evenodd"
d="M0 16.333c0 13.173 3.16 16.333 16.333 16.333 13.173 0 16.333-3.16 16.333-16.333C32.666 3.16 29.506 0 16.333 0 3.16 0 0 3.16 0 16.333Zm6.586 3.393L11.71 8.083c.865-1.962 2.528-3.027 4.624-3.027 2.096 0 3.759 1.065 4.624 3.027l5.123 11.643c.233.566.432 1.297.432 1.93 0 2.893-2.029 4.923-4.923 4.923-.986 0-1.769-.252-2.561-.506-.812-.261-1.634-.526-2.695-.526-1.048 0-1.89.267-2.718.529-.801.253-1.59.503-2.538.503-2.894 0-4.923-2.03-4.923-4.924 0-.632.2-1.363.432-1.929Zm9.747-9.613-5.056 11.443c1.497-.699 3.227-1.032 5.056-1.032 1.763 0 3.56.333 4.99 1.032l-4.99-11.444Z"
clip-rule="evenodd"
/>
</svg>
</a>
</div>
{{-- Bento with documentation, Adocasts, packages and Discord --}}
<div
class="isolate mt-10 max-w-screen-xl mx-auto px-16 xl:px-8 grid grid-cols-1 xl:grid-cols-2 xl:grid-rows-3 gap-8"
>
<article
class="row-span-3 relative p-6 shadow-sm hover:shadow border border-sand-7 hover:border-sand-8 rounded-2xl transition ease-in-out duration-700 group flex flex-col gap-8"
>
<div class="relative opacity-80">
<svg fill="none" viewBox="0 0 240 105">
<path fill="#F9F9F8" d="M0 4a4 4 0 0 1 4-4h232a4 4 0 0 1 4 4v101H0V4Z" />
<g fill="#000" fill-rule="evenodd" clip-path="url(#a)" clip-rule="evenodd">
<path
d="M24 11.444c0 4.391 1.053 5.445 5.444 5.445s5.445-1.054 5.445-5.445c0-4.39-1.054-5.444-5.445-5.444C25.054 6 24 7.053 24 11.444Zm2.195 1.131 1.708-3.88c.288-.655.843-1.01 1.541-1.01.699 0 1.253.355 1.542 1.01l1.707 3.88c.078.189.144.433.144.644 0 .964-.676 1.64-1.64 1.64-.33 0-.59-.083-.854-.168-.271-.087-.545-.175-.899-.175-.35 0-.63.089-.906.176-.267.085-.53.168-.846.168-.964 0-1.64-.677-1.64-1.641 0-.211.066-.455.143-.644Zm3.25-3.204-1.686 3.814c.499-.233 1.075-.344 1.685-.344.588 0 1.187.111 1.664.344l-1.664-3.814Zm26.473-.678c-.378 0-.65.268-.65.64 0 .374.272.641.65.641s.651-.267.651-.64-.273-.64-.65-.64Zm-11.907 5.502c-1.009 0-1.738-.745-1.738-1.91 0-1.187.73-1.933 1.737-1.933.468 0 .814.158 1.019.468V8.86h1.05v5.25h-1.05v-.372c-.2.304-.546.456-1.019.456Zm-.667-1.91c0-.652.352-1.077.887-1.077.54 0 .887.42.887 1.071 0 .64-.346 1.056-.887 1.056-.535 0-.887-.415-.887-1.05Zm4.384-.011c0-.646.351-1.06.877-1.06.53 0 .882.414.882 1.06 0 .646-.352 1.06-.883 1.06-.525 0-.876-.414-.876-1.06Zm11.571.835c0 .194-.147.31-.52.31-.42 0-.682-.221-.682-.489h-1.05c.026.725.714 1.265 1.711 1.265.946 0 1.55-.42 1.55-1.165 0-.557-.358-.945-1.066-1.087l-.762-.152c-.23-.047-.367-.163-.367-.315 0-.226.23-.347.525-.347.42 0 .583.195.583.426h.997c-.026-.683-.562-1.203-1.56-1.203-.929 0-1.559.468-1.559 1.176 0 .64.415.93 1.035 1.06l.756.164c.247.052.41.157.41.357Zm-2.85 1.002h-1.05v-3.675h1.05v3.675Zm-4.264-3.675v.384c.268-.31.625-.468 1.066-.468.824 0 1.36.536 1.36 1.365v2.394h-1.05v-2.173c0-.446-.252-.714-.688-.714-.436 0-.688.268-.688.714v2.173h-1.05v-3.675h1.05Zm-3.58-.084c-1.119 0-1.948.809-1.948 1.922s.83 1.921 1.948 1.921c1.123 0 1.953-.808 1.953-1.921s-.83-1.922-1.953-1.922Zm-8.758.856c-.535 0-.887.425-.887 1.076 0 .636.352 1.05.887 1.05.54 0 .887-.414.887-1.055 0-.65-.346-1.07-.887-1.07Zm-1.958 1.076c0 1.166.73 1.911 1.732 1.911.478 0 .82-.152 1.024-.456v.372h1.05v-3.675h-1.05v.384c-.21-.31-.556-.468-1.024-.468-1.003 0-1.732.746-1.732 1.932Z"
/>
</g>
<rect width="8" height="3" x="162" y="9.944" fill="#DAD9D6" rx="1" />
<rect width="14" height="3" x="174" y="9.944" fill="#DAD9D6" rx="1" />
<rect width="10" height="3" x="192" y="9.944" fill="#DAD9D6" rx="1" />
<rect width="10" height="3" x="206" y="9.944" fill="#DAD9D6" rx="1" />
<rect width="81" height="6" x="24" y="32" fill="#DAD9D6" rx="2" />
<rect width="95" height="6" x="24" y="44" fill="#DAD9D6" rx="2" />
<rect width="16" height="5" x="24" y="60" fill="#21201C" rx="1" />
<path fill="#DAD9D6" d="M24 85a4 4 0 0 1 4-4h184a4 4 0 0 1 4 4v20H24V85Z" />
<path fill="url(#b)" fill-opacity=".2" d="M24 85a4 4 0 0 1 4-4h184a4 4 0 0 1 4 4v20H24V85Z" />
<defs>
<linearGradient id="b" x1="120" x2="120" y1="81" y2="105" gradientUnits="userSpaceOnUse">
<stop stop-opacity="0" />
<stop offset="1" stop-color="#82827C" />
</linearGradient>
<clipPath id="a">
<path fill="#fff" d="M24 6h36.307v10.889H24z" />
</clipPath>
</defs>
</svg>
<div class="absolute left-0 right-0 bottom-0 h-16 bg-gradient-to-b from-white/0 to-white">
</div>
</div>
<div class="flex flex-row gap-4">
<div class="shrink-0 w-10 h-10 bg-primary/20 rounded-md flex justify-center items-center">
<svg class="h-6 w-6 fill-primary" viewBox="0 0 256 256">
<path
fill="currentColor"
d="M208 24H72a32 32 0 0 0-32 32v168a8 8 0 0 0 8 8h144a8 8 0 0 0 0-16H56a16 16 0 0 1 16-16h136a8 8 0 0 0 8-8V32a8 8 0 0 0-8-8m-88 16h48v72l-19.21-14.4a8 8 0 0 0-9.6 0L120 112Zm80 144H72a31.8 31.8 0 0 0-16 4.29V56a16 16 0 0 1 16-16h32v88a8 8 0 0 0 12.8 6.4L144 114l27.21 20.4A8 8 0 0 0 176 136a8 8 0 0 0 8-8V40h16Z"
/>
</svg>
</div>
<div class="space-y-1">
<h2 class="text-lg font-semibold">
<a href="https://docs.adonisjs.com" target="_blank">
<span>Documentation</span>
<span class="absolute inset-0"></span>
</a>
</h2>
<p class="text-sm text-sand-11 group-hover:text-sand-12 transition ease-in-out duration-700">
Dive into the official documentation to learn AdonisJS. Read carefully to discover an unmatched set of features, best practices and developer experience. Through examples, guides and API references, you'll find everything you need to build your next project. From installation to deployment, we've got you covered.
</p>
</div>
</div>
</article>
<article
class="relative p-6 shadow-sm hover:shadow border border-sand-7 hover:border-sand-8 rounded-2xl transition ease-in-out duration-700 group flex flex-row gap-4"
>
<div class="shrink-0 w-10 h-10 bg-primary/20 rounded-md flex justify-center items-center">
<svg class="h-6 w-6 fill-primary" viewBox="0 0 256 256">
<path
fill="currentColor"
d="m164.44 105.34-48-32A8 8 0 0 0 104 80v64a8 8 0 0 0 12.44 6.66l48-32a8 8 0 0 0 0-13.32M120 129.05V95l25.58 17ZM216 40H40a16 16 0 0 0-16 16v112a16 16 0 0 0 16 16h176a16 16 0 0 0 16-16V56a16 16 0 0 0-16-16m0 128H40V56h176zm16 40a8 8 0 0 1-8 8H32a8 8 0 0 1 0-16h192a8 8 0 0 1 8 8"
/>
</svg>
</div>
<div class="space-y-1">
<h2 class="text-lg font-semibold">
<a href="https://adocasts.com" target="_blank">
<span>Adocasts</span>
<span class="absolute inset-0"></span>
</a>
</h2>
<p class="text-sm text-sand-11 group-hover:text-sand-12 transition ease-in-out duration-700">
Level up your development and Adonis skills with hours of video content, from beginner to advanced, through databases, testing, and more.
</p>
</div>
</article>
<article
class="relative p-6 shadow-sm hover:shadow border border-sand-7 hover:border-sand-8 rounded-2xl transition ease-in-out duration-700 group flex flex-row gap-4"
>
<div class="shrink-0 w-10 h-10 bg-primary/20 rounded-md flex justify-center items-center">
<svg class="h-6 w-6 fill-primary" viewBox="0 0 256 256">
<path
fill="currentColor"
d="M208 96a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16h-32a16 16 0 0 0-16 16v8H96v-8a16 16 0 0 0-16-16H48a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h8v64h-8a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-8h64v8a16 16 0 0 0 16 16h32a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16h-8V96Zm-32-48h32v32h-32ZM48 48h32v15.9a.5.5 0 0 0 0 .2V80H48Zm32 160H48v-32h32v15.9a.5.5 0 0 0 0 .2zm128 0h-32v-32h32Zm-24-48h-8a16 16 0 0 0-16 16v8H96v-8a16 16 0 0 0-16-16h-8V96h8a16 16 0 0 0 16-16v-8h64v8a16 16 0 0 0 16 16h8Z"
/>
</svg>
</div>
<div class="space-y-1">
<h2 class="text-lg font-semibold">
<a href="https://packages.adonisjs.com" target="_blank">
<span>Packages</span>
<span class="absolute inset-0"></span>
</a>
</h2>
<p class="text-sm text-sand-11 group-hover:text-sand-12 transition ease-in-out duration-700">
Supercharge your AdonisJS application with packages built and maintained by both the core team and the community.
</p>
</div>
</article>
<article
class="relative p-6 shadow-sm hover:shadow border border-sand-7 hover:border-sand-8 rounded-2xl transition ease-in-out duration-700 group flex flex-row gap-4"
>
<div class="shrink-0 w-10 h-10 bg-primary/20 rounded-md flex justify-center items-center">
<svg class="h-6 w-6 fill-primary" viewBox="0 0 256 256">
<path
fill="currentColor"
d="M128 24a104 104 0 1 0 104 104A104.11 104.11 0 0 0 128 24m0 192a88 88 0 1 1 88-88 88.1 88.1 0 0 1-88 88m44.42-143.16-64 32a8.05 8.05 0 0 0-3.58 3.58l-32 64A8 8 0 0 0 80 184a8.1 8.1 0 0 0 3.58-.84l64-32a8.05 8.05 0 0 0 3.58-3.58l32-64a8 8 0 0 0-10.74-10.74M138 138l-40.11 20.11L118 118l40.15-20.07Z"
/>
</svg>
</div>
<div class="space-y-1">
<h2 class="text-lg font-semibold">
<a href="https://discord.gg/vDcEjq6" target="_blank">
<span>Discord</span>
<span class="absolute inset-0"></span>
</a>
</h2>
<p class="text-sm text-sand-11 group-hover:text-sand-12 transition ease-in-out duration-700">
Never get lost again, ask questions, and share your knowledge or projects with a growing and supportive community. Join us.
</p>
</div>
</article>
</div>
{{-- Features --}}
<div class="grow mt-10 mb-8 px-16 xl:px-8 max-w-screen-xl mx-auto">
<div class="grid grid-cols-1 sm:grid-cols-2 xl:grid-cols-4 gap-4">
<article
class="relative py-4 px-5 bg-white border border-transparent rounded-lg hover:border-sand-8 hover:shadow-sm transition duration-100 ease-in-out group"
>
<h2 class="font-semibold text-sand-12">
<a href="https://lucid.adonisjs.com" target="_blank" class="flex flex-row gap-2">
<span class="bg-[#D5EAE7] h-6 w-6 flex justify-center items-center rounded">
<svg class="h-4 w-4 fill-[#0E766E]" viewBox="0 0 24 24">
<g
fill="none"
stroke="currentColor"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
>
<path d="M4 6a8 3 0 1 0 16 0A8 3 0 1 0 4 6" />
<path d="M4 6v6a8 3 0 0 0 16 0V6" />
<path d="M4 12v6a8 3 0 0 0 16 0v-6" />
</g>
</svg>
</span>
<span>Lucid</span>
<span class="absolute inset-0"></span>
</a>
</h2>
<p class="mt-4 text-sm text-sand-11 group-hover:text-sand-12 transition ease-in-out duration-100">
A SQL ORM with a powerful query builder, active record, migrations, and model factories. Everything you need to work with databases.
</p>
<svg
class="absolute top-4 right-5 opacity-0 group-hover:opacity-100 text-sand-9 w-4 h-4 transition ease-in-out duration-100"
viewBox="0 0 24 24"
>
<path
fill="none"
stroke="currentColor"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M12 6H6a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2v-6m-7 1 9-9m-5 0h5v5"
/>
</svg>
</article>
<article
class="relative py-4 px-5 bg-white border border-transparent rounded-lg hover:border-sand-8 hover:shadow-sm transition duration-100 ease-in-out group"
>
<h2 class="font-semibold text-sand-12">
<a href="https://vinejs.dev/" target="_blank" class="flex flex-row gap-2">
<span class="bg-[#F3DBFC] h-6 w-6 flex justify-center items-center rounded">
<svg class="h-4 w-4 fill-[#CA5AF2]" viewBox="0 0 24 24">
<path
fill="none"
stroke="currentColor"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M12 3a12 12 0 0 0 8.5 3A12 12 0 0 1 12 21 12 12 0 0 1 3.5 6 12 12 0 0 0 12 3"
/>
</svg>
</span>
<span>Vine</span>
<span class="absolute inset-0"></span>
</a>
</h2>
<p class="mt-4 text-sm text-sand-11 group-hover:text-sand-12 transition ease-in-out duration-100">
A simple yet feature rich and type-safe form data validation. It comes with 50+ built-in rules and an expressive API to define custom rules.
</p>
<svg
class="absolute top-4 right-5 opacity-0 group-hover:opacity-100 text-sand-9 w-4 h-4 transition ease-in-out duration-100"
viewBox="0 0 24 24"
>
<path
fill="none"
stroke="currentColor"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M12 6H6a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2v-6m-7 1 9-9m-5 0h5v5"
/>
</svg>
</article>
<article
class="relative py-4 px-5 bg-white border border-transparent rounded-lg hover:border-sand-8 hover:shadow-sm transition duration-100 ease-in-out group"
>
<h2 class="font-semibold text-sand-12">
<a href="https://edgejs.dev/" target="_blank" class="flex flex-row gap-2">
<span class="bg-[#B8EAE0] h-6 w-6 flex justify-center items-center rounded">
<svg class="h-4 w-4 fill-[#4BBBA5]" viewBox="0 0 24 24">
<path
fill="none"
stroke="currentColor"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M7 4a2 2 0 0 0-2 2v3a2 3 0 0 1-2 3 2 3 0 0 1 2 3v3a2 2 0 0 0 2 2M17 4a2 2 0 0 1 2 2v3a2 3 0 0 0 2 3 2 3 0 0 0-2 3v3a2 2 0 0 1-2 2"
/>
</svg>
</span>
<span>Edge</span>
<span class="absolute inset-0"></span>
</a>
</h2>
<p class="mt-4 text-sm text-sand-11 group-hover:text-sand-12 transition ease-in-out duration-100">
Write your views with ease and enjoy the power of a simple, modern and battteries included template engine. You'll love it.
</p>
<svg
class="absolute top-4 right-5 opacity-0 group-hover:opacity-100 text-sand-9 w-4 h-4 transition ease-in-out duration-100"
viewBox="0 0 24 24"
>
<path
fill="none"
stroke="currentColor"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M12 6H6a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2v-6m-7 1 9-9m-5 0h5v5"
/>
</svg>
</article>
<article
class="relative py-4 px-5 bg-white border border-transparent rounded-lg hover:border-sand-8 hover:shadow-sm transition duration-100 ease-in-out group"
>
<h2 class="font-semibold text-sand-12">
<a href="https://japa.dev" target="_blank" class="flex flex-row gap-2">
<span class="bg-[#FACDDC] h-6 w-6 flex justify-center items-center rounded">
<svg class="h-4 w-4 fill-[#DD3074]" viewBox="0 0 256 256">
<path
fill="currentColor"
d="m240.49 83.51-60-60a12 12 0 0 0-17 0L34.28 152.75a48.77 48.77 0 0 0 69 69l111.2-111.26 21.31-7.11a12 12 0 0 0 4.7-19.87M86.28 204.75a24.77 24.77 0 0 1-35-35l28.13-28.13c7.73-2.41 19.58-3 35.06 5a84 84 0 0 0 21.95 8ZM204.2 88.62a12.15 12.15 0 0 0-4.69 2.89l-38.89 38.9c-7.73 2.41-19.58 3-35.06-5a84 84 0 0 0-21.94-8L172 49l37.79 37.79Z"
/>
</svg>
</span>
<span>Japa</span>
<span class="absolute inset-0"></span>
</a>
</h2>
<p class="mt-4 text-sm text-sand-11 group-hover:text-sand-12 transition ease-in-out duration-100">
From JSON API tests using Open API schema to browser tests with Playwright, it comes with everything you need to test your application.
</p>
<svg
class="absolute top-4 right-5 opacity-0 group-hover:opacity-100 text-sand-9 w-4 h-4 transition ease-in-out duration-100"
viewBox="0 0 24 24"
>
<path
fill="none"
stroke="currentColor"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M12 6H6a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2v-6m-7 1 9-9m-5 0h5v5"
/>
</svg>
</article>
</div>
</div>
<div
class="text-sm text-center [&>code]:font-medium [&>code]:text-[#a599ff] bg-sand-12 text-sand-1 relative py-2"
>
Route for this page is registered in <code>start/routes.ts</code> file, rendering <code>resources/views/pages/home.edge</code> template
</div>
</div>
</body>
</html>

46
scripts/setup_database.ts Normal file
View File

@ -0,0 +1,46 @@
import pg from 'pg'
import env from '#start/env'
const { Client } = pg
async function setupDatabase() {
const client = new Client({
host: env.get('DB_HOST'),
port: env.get('DB_PORT'),
user: env.get('DB_USER'),
password: env.get('DB_PASSWORD'),
database: 'postgres',
})
try {
await client.connect()
console.log('Connected to PostgreSQL server')
const dbName = env.get('DB_DATABASE')
const checkDbQuery = `SELECT 1 FROM pg_database WHERE datname = $1`
const result = await client.query(checkDbQuery, [dbName])
if (result.rows.length === 0) {
console.log(`Database "${dbName}" does not exist. Creating...`)
await client.query(`CREATE DATABASE ${dbName}`)
console.log(`Database "${dbName}" created successfully`)
} else {
console.log(`Database "${dbName}" already exists`)
}
} catch (error) {
console.error('Error setting up database:', error)
throw error
} finally {
await client.end()
}
}
setupDatabase()
.then(() => {
console.log('Database setup completed successfully')
process.exit(0)
})
.catch((error) => {
console.error('Database setup failed:', error)
process.exit(1)
})

45
start/env.ts Normal file
View File

@ -0,0 +1,45 @@
/*
|--------------------------------------------------------------------------
| Environment variables service
|--------------------------------------------------------------------------
|
| The `Env.create` method creates an instance of the Env service. The
| service validates the environment variables and also cast values
| to JavaScript data types.
|
*/
import { Env } from '@adonisjs/core/env'
export default await Env.create(new URL('../', import.meta.url), {
NODE_ENV: Env.schema.enum(['development', 'production', 'test'] as const),
PORT: Env.schema.number(),
APP_KEY: Env.schema.string(),
HOST: Env.schema.string({ format: 'host' }),
LOG_LEVEL: Env.schema.string(),
/*
|----------------------------------------------------------
| Variables for configuring session package
|----------------------------------------------------------
*/
SESSION_DRIVER: Env.schema.enum(['cookie', 'memory'] as const),
/*
|----------------------------------------------------------
| Variables for configuring database connection
|----------------------------------------------------------
*/
DB_HOST: Env.schema.string({ format: 'host' }),
DB_PORT: Env.schema.number(),
DB_USER: Env.schema.string(),
DB_PASSWORD: Env.schema.string.optional(),
DB_DATABASE: Env.schema.string(),
/*
|----------------------------------------------------------
| Variables for Telegram bot
|----------------------------------------------------------
*/
TELEGRAM_BOT_TOKEN: Env.schema.string(),
})

49
start/kernel.ts Normal file
View File

@ -0,0 +1,49 @@
/*
|--------------------------------------------------------------------------
| HTTP kernel file
|--------------------------------------------------------------------------
|
| The HTTP kernel file is used to register the middleware with the server
| or the router.
|
*/
import router from '@adonisjs/core/services/router'
import server from '@adonisjs/core/services/server'
/**
* The error handler is used to convert an exception
* to an HTTP response.
*/
server.errorHandler(() => import('#exceptions/handler'))
/**
* The server middleware stack runs middleware on all the HTTP
* requests, even if there is no route registered for
* the request URL.
*/
server.use([
() => import('#middleware/container_bindings_middleware'),
() => import('@adonisjs/static/static_middleware'),
() => import('@adonisjs/vite/vite_middleware'),
])
/**
* The router middleware stack runs middleware on all the HTTP
* requests with a registered route.
*/
router.use([
() => import('@adonisjs/core/bodyparser_middleware'),
() => import('@adonisjs/session/session_middleware'),
() => import('@adonisjs/shield/shield_middleware'),
() => import('@adonisjs/auth/initialize_auth_middleware')
])
/**
* Named middleware collection must be explicitly assigned to
* the routes or the routes group.
*/
export const middleware = router.named({
guest: () => import('#middleware/guest_middleware'),
auth: () => import('#middleware/auth_middleware')
})

51
start/routes.ts Normal file
View File

@ -0,0 +1,51 @@
/*
|--------------------------------------------------------------------------
| Routes file
|--------------------------------------------------------------------------
|
| The routes file is used for defining the HTTP routes.
|
*/
import router from '@adonisjs/core/services/router'
const KeywordsController = () => import('#controllers/keywords_controller')
const AuctionsController = () => import('#controllers/auctions_controller')
const ParseLogsController = () => import('#controllers/parse_logs_controller')
router.on('/').render('pages/home')
// Health check endpoint for Docker
router.get('/health', async ({ response }) => {
return response.ok({ status: 'ok', timestamp: new Date().toISOString() })
})
// List recent auctions (last 3 days)
router.get('/list', [AuctionsController, 'list'])
// HTML view of recent auctions (last 3 days)
router.get('/list-view', [AuctionsController, 'listView'])
// Trigger auction parsing
router.post('/trigger-parse', [AuctionsController, 'triggerParse'])
/*
|--------------------------------------------------------------------------
| API Routes
|--------------------------------------------------------------------------
*/
router.group(() => {
// Keywords
router.get('/keywords', [KeywordsController, 'index'])
router.post('/keywords', [KeywordsController, 'store'])
router.delete('/keywords/:id', [KeywordsController, 'destroy'])
// Auctions
router.get('/auctions/search', [AuctionsController, 'search'])
router.get('/auctions/:id', [AuctionsController, 'show'])
router.get('/auctions', [AuctionsController, 'index'])
// Parse Logs
router.get('/parse-logs', [ParseLogsController, 'index'])
}).prefix('/api')

56
start/scheduler.ts Normal file
View File

@ -0,0 +1,56 @@
/**
* Scheduler configuration
*
* This file defines all scheduled tasks for the application.
* The scheduler runs in a separate process using: node ace scheduler:work
*
* Learn more: https://github.com/kabbouchi/adonisjs-scheduler
*/
import scheduler from 'adonisjs-scheduler/services/main'
import ParseAuctions from '../commands/parse_auctions.js'
/**
* Schedule auction parsing every 6 hours
*
* This task will:
* 1. Scrape auctions from icetrade.by
* 2. Upsert them to the database
* 3. Create notifications for keyword matches
*
* Configuration:
* - Runs every 6 hours
* - Prevents overlapping executions
* - Scrapes 1 page by default (can be adjusted)
*/
scheduler
.command(ParseAuctions, ['--pages=1'])
.everySixHours()
.withoutOverlapping()
/**
* Alternative schedule options (uncomment to use):
*
* Run every hour:
*/
// scheduler.command(ParseAuctions).everyHour().withoutOverlapping()
/**
* Run every 3 hours using cron:
*/
// scheduler.command(ParseAuctions).cron('0 *\/3 * * *').withoutOverlapping()
/**
* Run at specific times (6am, 12pm, 6pm, 12am):
*/
// scheduler.command(ParseAuctions).cron('0 6,12,18,0 * * *').withoutOverlapping()
/**
* Run every 30 minutes:
*/
// scheduler.command(ParseAuctions).everyThirtyMinutes().withoutOverlapping()
/**
* Run daily at 3am:
*/
// scheduler.command(ParseAuctions).dailyAt('3:00').withoutOverlapping()

37
tests/bootstrap.ts Normal file
View File

@ -0,0 +1,37 @@
import { assert } from '@japa/assert'
import app from '@adonisjs/core/services/app'
import type { Config } from '@japa/runner/types'
import { pluginAdonisJS } from '@japa/plugin-adonisjs'
import testUtils from '@adonisjs/core/services/test_utils'
/**
* This file is imported by the "bin/test.ts" entrypoint file
*/
/**
* Configure Japa plugins in the plugins array.
* Learn more - https://japa.dev/docs/runner-config#plugins-optional
*/
export const plugins: Config['plugins'] = [assert(), pluginAdonisJS(app)]
/**
* Configure lifecycle function to run before and after all the
* tests.
*
* The setup functions are executed before all the tests
* The teardown functions are executed after all the tests
*/
export const runnerHooks: Required<Pick<Config, 'setup' | 'teardown'>> = {
setup: [],
teardown: [],
}
/**
* Configure suites by tapping into the test suite instance.
* Learn more - https://japa.dev/docs/test-suites#lifecycle-hooks
*/
export const configureSuite: Config['configureSuite'] = (suite) => {
if (['browser', 'functional', 'e2e'].includes(suite.name)) {
return suite.setup(() => testUtils.httpServer().start())
}
}

11
tests/fixtures/empty_page.html vendored Normal file
View File

@ -0,0 +1,11 @@
<!DOCTYPE html>
<html>
<head>
<title>Empty Page</title>
</head>
<body>
<div class="content">
<p>No auctions found.</p>
</div>
</body>
</html>

36
tests/fixtures/malformed_auction.html vendored Normal file
View File

@ -0,0 +1,36 @@
<!DOCTYPE html>
<html>
<head>
<title>Malformed Auction Data</title>
</head>
<body>
<table class="auctions w100">
<thead>
<tr>
<th></th>
<th>Название</th>
</tr>
</thead>
<tbody>
<tr>
<td></td>
<td>
<a href="not-a-valid-url">Missing Auction Number</a>
</td>
<td></td>
<td></td>
<td></td>
</tr>
<tr>
<td>AUC-2024-999</td>
<td>
<a>No href attribute</a>
</td>
<td>Test Org</td>
<td>Active</td>
<td></td>
</tr>
</tbody>
</table>
</body>
</html>

49
tests/fixtures/sample_auctions.html vendored Normal file
View File

@ -0,0 +1,49 @@
<!DOCTYPE html>
<html>
<head>
<title>Test Auctions Page</title>
</head>
<body>
<table class="auctions w100">
<thead>
<tr>
<th></th>
<th>Название</th>
<th>Организация</th>
<th>Статус</th>
<th>Срок</th>
</tr>
</thead>
<tbody>
<tr>
<td>AUC-2024-001</td>
<td>
<a href="/trades/view/12345">Поставка офисной мебели</a>
<span class="description">Комплект офисной мебели для нового офиса</span>
</td>
<td>ООО "Тестовая Компания"</td>
<td>Активный</td>
<td>15.03.2025</td>
</tr>
<tr>
<td>AUC-2024-002</td>
<td>
<a href="/trades/view/12346">Закупка компьютерного оборудования</a>
</td>
<td>ГУО "Беларусь"</td>
<td>Завершен</td>
<td>01.02.2025</td>
</tr>
<tr>
<td>AUC-2024-003</td>
<td>
<a href="https://icetrade.by/trades/view/12347">Услуги по ремонту помещений</a>
</td>
<td>ЗАО "Строй-Сервис"</td>
<td>Подготовка</td>
<td></td>
</tr>
</tbody>
</table>
</body>
</html>

View File

@ -0,0 +1,274 @@
import { test } from '@japa/runner'
import { ScraperService, ScraperError } from '#services/scraper_service'
import { readFile } from 'node:fs/promises'
import { fileURLToPath } from 'node:url'
import { dirname, join } from 'node:path'
// Get the directory path for fixtures
const __filename = fileURLToPath(import.meta.url)
const __dirname = dirname(__filename)
const fixturesPath = join(__dirname, '..', '..', 'fixtures')
test.group('ScraperService - parsePage', () => {
test('should parse valid auction HTML correctly', async ({ assert }) => {
const scraper = new ScraperService()
const html = await readFile(join(fixturesPath, 'sample_auctions.html'), 'utf-8')
const auctions = scraper.parsePage(html)
assert.lengthOf(auctions, 3, 'Should parse all 3 auctions')
// Verify first auction
const firstAuction = auctions[0]
assert.equal(firstAuction.auctionNum, 'AUC-2024-001')
assert.equal(firstAuction.title, 'Поставка офисной мебели')
assert.equal(firstAuction.organization, 'ООО "Тестовая Компания"')
assert.equal(firstAuction.status, 'Активный')
assert.equal(firstAuction.deadline, '15.03.2025')
assert.equal(firstAuction.link, 'https://icetrade.by/trades/view/12345')
assert.equal(firstAuction.description, 'Комплект офисной мебели для нового офиса')
})
test('should handle relative URLs and convert to absolute', async ({ assert }) => {
const scraper = new ScraperService()
const html = await readFile(join(fixturesPath, 'sample_auctions.html'), 'utf-8')
const auctions = scraper.parsePage(html)
// First two auctions have relative URLs
assert.isTrue(auctions[0].link.startsWith('https://icetrade.by/'))
assert.isTrue(auctions[1].link.startsWith('https://icetrade.by/'))
// Third auction already has absolute URL
assert.equal(auctions[2].link, 'https://icetrade.by/trades/view/12347')
})
test('should handle null deadline gracefully', async ({ assert }) => {
const scraper = new ScraperService()
const html = await readFile(join(fixturesPath, 'sample_auctions.html'), 'utf-8')
const auctions = scraper.parsePage(html)
// Third auction has empty deadline
assert.isNull(auctions[2].deadline)
})
test('should handle null description when not present', async ({ assert }) => {
const scraper = new ScraperService()
const html = await readFile(join(fixturesPath, 'sample_auctions.html'), 'utf-8')
const auctions = scraper.parsePage(html)
// Second auction has no description
assert.isNull(auctions[1].description)
})
test('should return empty array when no auctions table found', async ({ assert }) => {
const scraper = new ScraperService()
const html = await readFile(join(fixturesPath, 'empty_page.html'), 'utf-8')
const auctions = scraper.parsePage(html)
assert.lengthOf(auctions, 0, 'Should return empty array')
})
test('should skip invalid auction rows and continue parsing', async ({ assert }) => {
const scraper = new ScraperService()
const html = await readFile(join(fixturesPath, 'malformed_auction.html'), 'utf-8')
const auctions = scraper.parsePage(html)
// Both rows in malformed_auction.html should fail validation
// First row: empty auction number
// Second row: no href (empty string fails URL validation)
assert.lengthOf(auctions, 0, 'Should skip all malformed rows')
})
test('should throw ScraperError for completely invalid HTML', ({ assert }) => {
const scraper = new ScraperService()
const invalidHtml = 'This is not valid HTML at all {{{'
// Should not throw - cheerio is very lenient
// But should return empty array since no table found
const auctions = scraper.parsePage(invalidHtml)
assert.lengthOf(auctions, 0)
})
})
test.group('ScraperService - fetchPage', () => {
test('should build correct URL with all parameters', ({ assert }) => {
const scraper = new ScraperService()
// Access private method via type assertion for testing
const buildUrl = (scraper as any).buildUrl.bind(scraper)
const url = buildUrl(1)
assert.include(url, 'https://icetrade.by/trades/index')
assert.include(url, 'p=1')
assert.include(url, 'onPage=100')
assert.include(url, 'sort=num%3Adesc')
assert.include(url, 'zakup_type%5B1%5D=1')
assert.include(url, 'zakup_type%5B2%5D=1')
assert.include(url, 'r%5B1%5D=1')
assert.include(url, 't%5BTrade%5D=1')
})
test('should include correct page number in URL', ({ assert }) => {
const scraper = new ScraperService()
const buildUrl = (scraper as any).buildUrl.bind(scraper)
const url1 = buildUrl(1)
const url5 = buildUrl(5)
assert.include(url1, 'p=1')
assert.include(url5, 'p=5')
})
})
test.group('ScraperService - scrapeAuctions', () => {
test('should throw error if maxPages is less than 1', async ({ assert }) => {
const scraper = new ScraperService()
await assert.rejects(
async () => await scraper.scrapeAuctions(0),
'maxPages must be at least 1'
)
await assert.rejects(
async () => await scraper.scrapeAuctions(-5),
'maxPages must be at least 1'
)
})
// Note: Full integration tests for scrapeAuctions would require:
// 1. Mocking fetch API
// 2. Providing mock responses
// These should be in integration tests, not unit tests
})
test.group('ScraperService - validation', () => {
test('should validate auction number is not empty', ({ assert }) => {
const scraper = new ScraperService()
const html = `
<table class="auctions w100">
<tbody>
<tr>
<td> </td>
<td><a href="https://test.com">Title</a></td>
<td>Org</td>
<td>Status</td>
<td></td>
</tr>
</tbody>
</table>
`
const auctions = scraper.parsePage(html)
assert.lengthOf(auctions, 0, 'Should skip auction with empty number')
})
test('should validate title is not empty', ({ assert }) => {
const scraper = new ScraperService()
const html = `
<table class="auctions w100">
<tbody>
<tr>
<td>AUC-001</td>
<td><a href="https://test.com"> </a></td>
<td>Org</td>
<td>Status</td>
<td></td>
</tr>
</tbody>
</table>
`
const auctions = scraper.parsePage(html)
assert.lengthOf(auctions, 0, 'Should skip auction with empty title')
})
test('should validate organization is not empty', ({ assert }) => {
const scraper = new ScraperService()
const html = `
<table class="auctions w100">
<tbody>
<tr>
<td>AUC-001</td>
<td><a href="https://test.com">Title</a></td>
<td> </td>
<td>Status</td>
<td></td>
</tr>
</tbody>
</table>
`
const auctions = scraper.parsePage(html)
assert.lengthOf(auctions, 0, 'Should skip auction with empty organization')
})
test('should validate link is a valid URL', ({ assert }) => {
const scraper = new ScraperService()
const html = `
<table class="auctions w100">
<tbody>
<tr>
<td>AUC-001</td>
<td><a href="">Title</a></td>
<td>Org</td>
<td>Status</td>
<td></td>
</tr>
</tbody>
</table>
`
const auctions = scraper.parsePage(html)
assert.lengthOf(auctions, 0, 'Should skip auction with invalid URL')
})
test('should trim whitespace from all fields', ({ assert }) => {
const scraper = new ScraperService()
const html = `
<table class="auctions w100">
<tbody>
<tr>
<td> AUC-001 </td>
<td><a href="https://test.com"> Test Title </a></td>
<td> Test Org </td>
<td> Active </td>
<td> 2025-03-15 </td>
</tr>
</tbody>
</table>
`
const auctions = scraper.parsePage(html)
assert.lengthOf(auctions, 1)
const auction = auctions[0]
assert.equal(auction.auctionNum, 'AUC-001')
assert.equal(auction.title, 'Test Title')
assert.equal(auction.organization, 'Test Org')
assert.equal(auction.status, 'Active')
assert.equal(auction.deadline, '2025-03-15')
})
})
test.group('ScraperService - ScraperError', () => {
test('should create ScraperError with message', ({ assert }) => {
const error = new ScraperError('Test error message')
assert.equal(error.message, 'Test error message')
assert.equal(error.name, 'ScraperError')
assert.instanceOf(error, Error)
})
test('should create ScraperError with cause', ({ assert }) => {
const originalError = new Error('Original error')
const error = new ScraperError('Wrapped error', originalError)
assert.equal(error.message, 'Wrapped error')
assert.equal(error.cause, originalError)
})
})

7
tsconfig.json Normal file
View File

@ -0,0 +1,7 @@
{
"extends": "@adonisjs/tsconfig/tsconfig.app.json",
"compilerOptions": {
"rootDir": "./",
"outDir": "./build"
}
}

19
vite.config.ts Normal file
View File

@ -0,0 +1,19 @@
import { defineConfig } from 'vite'
import adonisjs from '@adonisjs/vite/client'
export default defineConfig({
plugins: [
adonisjs({
/**
* Entrypoints of your application. Each entrypoint will
* result in a separate bundle.
*/
entrypoints: ['resources/css/app.css', 'resources/js/app.js'],
/**
* Paths to watch and reload the browser on file change
*/
reload: ['resources/views/**/*.edge'],
}),
],
})