External Site Price and Stock Monitoring Implementation
Price monitoring of external sites is needed in two scenarios: tracking distributors (ensuring recommended prices aren't broken) and tracking specific SKU competitors for quick reaction. In both cases, you need a system that regularly checks specified URLs and alerts on deviations.
System Structure
URL List → Scheduler → Fetcher → Parser → Comparator → Alert Engine
↓
Snapshot Store
Key feature: system stores history of values, not just current state. This allows building price change charts and seeing patterns.
Data Model
CREATE TABLE watch_targets (
id BIGSERIAL PRIMARY KEY,
url TEXT NOT NULL UNIQUE,
label VARCHAR(255), -- "DNS: Samsung S24 256GB"
our_product_id BIGINT REFERENCES products(id),
site_id INT REFERENCES external_sites(id),
check_interval INTERVAL DEFAULT '4 hours',
price_selector VARCHAR(500),
stock_selector VARCHAR(500),
price_type VARCHAR(20) DEFAULT 'text', -- 'text', 'attr', 'json', 'meta'
price_attr VARCHAR(100), -- for type=attr or meta
price_regex VARCHAR(255), -- additional cleanup via regex
alert_threshold_pct NUMERIC(5,2) DEFAULT 5.0, -- alert on change > N%
is_active BOOLEAN DEFAULT TRUE,
last_checked_at TIMESTAMP,
last_price NUMERIC(12,2),
last_in_stock BOOLEAN
);
CREATE TABLE watch_snapshots (
id BIGSERIAL PRIMARY KEY,
target_id BIGINT REFERENCES watch_targets(id) ON DELETE CASCADE,
price NUMERIC(12,2),
in_stock BOOLEAN,
raw_price_text VARCHAR(200),
http_status SMALLINT,
error TEXT,
captured_at TIMESTAMP DEFAULT NOW()
);
CREATE INDEX idx_snapshots_target_time ON watch_snapshots(target_id, captured_at DESC);
Flexible Price Parser
Different sites store prices differently. Parser supports several modes:
class FlexiblePriceExtractor
{
public function extract(string $html, WatchTarget $target): ?ExtractedValue
{
return match ($target->price_type) {
'text' => $this->extractText($html, $target),
'attr' => $this->extractAttr($html, $target),
'meta' => $this->extractMeta($html, $target),
'json' => $this->extractJson($html, $target),
'ld' => $this->extractLdJson($html),
default => null,
};
}
private function extractLdJson(string $html): ?ExtractedValue
{
// Schema.org Product markup—universal for many stores
$crawler = new Crawler($html);
$nodes = $crawler->filter('script[type="application/ld+json"]');
foreach ($nodes as $node) {
$data = json_decode($node->textContent, true);
if (!$data) continue;
$type = $data['@type'] ?? $data[0]['@type'] ?? null;
if (!in_array($type, ['Product', 'Offer'])) continue;
$offer = $data['offers'] ?? $data;
if (is_array($offer) && isset($offer[0])) $offer = $offer[0];
$price = $offer['price'] ?? null;
$inStock = ($offer['availability'] ?? '') === 'https://schema.org/InStock';
if ($price !== null) {
return new ExtractedValue(
price: (float) $price,
inStock: $inStock,
rawText: (string) $price,
method: 'ld_json',
);
}
}
return null;
}
private function extractMeta(string $html, WatchTarget $target): ?ExtractedValue
{
// Open Graph / meta tags: <meta property="product:price:amount" content="29990">
$crawler = new Crawler($html);
$selector = "meta[property='{$target->price_attr}'], meta[name='{$target->price_attr}']";
try {
$content = $crawler->filter($selector)->attr('content');
return $this->parseNumeric($content);
} catch (\Exception $e) {
return null;
}
}
private function extractText(string $html, WatchTarget $target): ?ExtractedValue
{
if (!$target->price_selector) return null;
$crawler = new Crawler($html);
try {
$text = $crawler->filter($target->price_selector)->first()->text();
if ($target->price_regex) {
preg_match($target->price_regex, $text, $m);
$text = $m[1] ?? $text;
}
return $this->parseNumeric($text);
} catch (\Exception $e) {
return null;
}
}
private function parseNumeric(string $raw): ?ExtractedValue
{
$clean = preg_replace('/[^\d.,]/', '', $raw);
$clean = str_replace(',', '.', $clean);
// "29.990" (thousand separator as dot) → "29990"
if (preg_match('/^\d{1,3}\.\d{3}$/', $clean)) {
$clean = str_replace('.', '', $clean);
}
if (!is_numeric($clean) || (float) $clean <= 0) return null;
return new ExtractedValue(price: (float) $clean, rawText: $raw);
}
}
Watch Target Check Job
class CheckWatchTargetJob implements ShouldQueue
{
public int $timeout = 30;
public int $tries = 2;
public function handle(FlexiblePriceExtractor $extractor, WatchAlertService $alerts): void
{
$target = WatchTarget::findOrFail($this->targetId);
// Fetch
$response = $this->fetch($target->url);
if (!$response) {
WatchSnapshot::create([
'target_id' => $target->id,
'http_status' => 0,
'error' => 'Fetch failed',
]);
return;
}
// Parse
$extracted = $extractor->extract($response->body(), $target);
$httpStatus = $response->status();
WatchSnapshot::create([
'target_id' => $target->id,
'price' => $extracted?->price,
'in_stock' => $extracted?->inStock,
'raw_price_text' => $extracted?->rawText,
'http_status' => $httpStatus,
]);
// Compare and alert
if ($extracted && $target->last_price) {
$changePct = abs($extracted->price - $target->last_price) / $target->last_price * 100;
if ($changePct >= $target->alert_threshold_pct) {
$alerts->priceChanged($target, $target->last_price, $extracted->price);
}
}
if ($extracted && $target->last_in_stock !== null && $extracted->inStock !== $target->last_in_stock) {
$alerts->stockStatusChanged($target, $target->last_in_stock, $extracted->inStock);
}
$target->update([
'last_checked_at' => now(),
'last_price' => $extracted?->price ?? $target->last_price,
'last_in_stock' => $extracted?->inStock ?? $target->last_in_stock,
]);
}
}
Check Schedule Dispatcher
class WatchScheduler
{
public function dispatch(): void
{
WatchTarget::active()
->where(function ($q) {
$q->whereNull('last_checked_at')
->orWhereRaw("last_checked_at + check_interval < NOW()");
})
->orderBy('last_checked_at')
->chunk(200, function ($targets) {
foreach ($targets as $target) {
CheckWatchTargetJob::dispatch($target->id)
->onQueue('monitoring');
}
});
}
}
Monitoring Management Interface
In admin panel:
- List of URLs with current price and last check time
- "Check now" button
- 30-day price change chart
- Alert threshold settings per URL
- Bulk add URLs from CSV
Notification Patterns
class WatchAlertService
{
public function priceChanged(WatchTarget $target, float $oldPrice, float $newPrice): void
{
$direction = $newPrice < $oldPrice ? '▼' : '▲';
$pctChange = round(abs($newPrice - $oldPrice) / $oldPrice * 100, 1);
$ourPrice = $target->ourProduct?->price;
$text = "{$direction} *Price change* on {$target->site->name}\n"
. "{$target->label}\n"
. "Was: " . number_format($oldPrice, 0, '.', ' ') . " rub.\n"
. "Now: " . number_format($newPrice, 0, '.', ' ') . " rub. ({$pctChange}%)\n";
if ($ourPrice) {
$diff = round(($newPrice - $ourPrice) / $ourPrice * 100, 1);
$text .= "Our price: " . number_format($ourPrice, 0, '.', ' ') . " rub. "
. ($diff > 0 ? "(we're cheaper by {$diff}%)" : "(they're cheaper by " . abs($diff) . "%)") . "\n";
}
$text .= "\n[Open page]({$target->url})";
$this->telegram->sendMessage([
'chat_id' => config('telegram.price_watch_chat'),
'text' => $text,
'parse_mode' => 'Markdown',
]);
}
}
Timeline
- Data schema + FlexiblePriceExtractor + LD-JSON: 1–2 days
- CheckWatchTargetJob + dispatcher: 0.5 days
- Telegram notifications: 0.5 days
- Management interface + charts: 1 day
- Playwright adapter for JS sites: +1 day
Total: 3–4 working days.