Integrate Playwright for improved web scraping by launching a headless browser, enhancing product data retrieval with better error handling and dynamic content loading. Update logging for price saving confirmation.

index.js

@@ -1,33 +1,32 @@
 const cheerio = require("cheerio");
 const cron = require("node-cron");
 const axios = require("axios");
-const { chromium } = require("playwright");
 const fs = require("fs");
+const { chromium } = require("playwright");
 
 const discordWebhook =
   "https://discord.com/api/webhooks/1439286509390921749/t2Hb8XloF6zhDRYD1yh_QlkHHa9eHUyXvd9TxZRHwqR_b_OxxbnwDgsm4em8TwA9NQIa";
 
-const priceSelectors = {
+function sendMessage(message) {
-  miodowamydlarnia: ".projector_prices__price",
+  axios.post(discordWebhook, {
-  greentouch: ".main-price",
+    content: message,
-  amazon: "#corePrice_feature_div .a-price .a-offscreen",
+  });
-  soxo: "#projector_price_value span",
+}
-  empik: '[data-ta-section="priceMainContainer"] [data-ta="price"]',
-  notino: "#pd-price",
-};
-
-const excludePage = ["allegro", "homla.com.pl", "home-you.com"];
 
+let isFirstRun = true;
 async function compareAndSave(productsPrice) {
-  const productsPriceJson =
+  if (!fs.existsSync("productsPrice.json")) {
-    fs.readFileSync("productsPrice.json", "utf8") || "[]";
+    fs.writeFileSync("productsPrice.json", "[]");
+  }
+
+  const productsPriceJson = fs.readFileSync("productsPrice.json", "utf8");
   const oldProductsPrice = JSON.parse(productsPriceJson);
 
   const diffProducts = [];
 
   for (const product of productsPrice) {
     const oldProduct = oldProductsPrice.find(
-      (oldProduct) => oldProduct.link === product.link
+      (oldProduct) => oldProduct.name === product.name
     );
 
     if (oldProduct && oldProduct.price !== product.price) {
@@ -36,20 +35,22 @@ async function compareAndSave(productsPrice) {
         newPrice: product.price,
         oldPrice: oldProduct.price,
         link: product.link,
+        shop: product.shop,
       });
     }
   }
 
   for (const product of diffProducts) {
-    await axios.post(discordWebhook, {
+    sendMessage(
-      content: `Zmiana ceny **${product.name}**:\nCena: ${product.oldPrice} -> ${product.newPrice}\nLink: ${product.link}`,
+      `Zmiana ceny **${product.name}**: ${product.oldPrice} -> ${product.newPrice}\nLink: ${product.link}`
-    });
+    );
   }
 
-  if (diffProducts.length === 0) {
+  if (isFirstRun) {
-    await axios.post(discordWebhook, {
+    for (const product of productsPrice) {
-      content: "Brak zmian w cenach",
+      sendMessage(`Początkowa cena **${product.name}**: ${product.price}`);
-    });
+    }
+    isFirstRun = false;
   }
 
   fs.writeFileSync(
@@ -61,82 +62,81 @@ async function compareAndSave(productsPrice) {
 async function getProducts() {
   const products = await axios
     .get(
-      "https://db.maciaszek.ovh/api/collections/gifts_items/records?fields=title,link"
+      "https://db.maciaszek.ovh/api/collections/gifts_items/records?fields=ceneo_id&filter=ceneo_id!=''"
     )
-    .then((response) => response.data.items);
+    .then((response) => response.data.items.map((item) => item.ceneo_id));
 
-  return products
+  return products;
-    .filter((product) => product.link !== "")
-    .filter(
-      (product) => !excludePage.some((page) => product.link.includes(page))
-    );
 }
 
 async function init() {
-  const productsWithLinks = await getProducts();
+  const browser = await chromium.launch({ headless: true });
-  const productsWithBrowser = [];
+  const context = await browser.newContext({
-  const productsPrice = [];
+    userAgent:
-  const selectors = Object.keys(priceSelectors);
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  });
+  const page = await context.newPage();
 
-  for (const product of productsWithLinks) {
+  const productsIds = await getProducts();
-    if (product.link === "") continue;
+  const products = [];
 
+  for (const productId of productsIds) {
     try {
-      const { data } = await axios.get(product.link, {
+      await page.goto(`https://www.ceneo.pl/${productId}`, {
-        headers: {
+        waitUntil: "networkidle",
-          "User-Agent":
+        timeout: 30000,
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
-        },
       });
 
-      const $ = cheerio.load(data);
+      // Wait for bot detection to pass and content to load
-      const selector = selectors.find((selector) =>
+      await page
-        product.link.includes(selector)
+        .waitForSelector(".product-offer__container, .product-top", {
-      );
+          timeout: 10000,
-      const price = $(priceSelectors[selector]).text();
+        })
-      productsPrice.push({
+        .catch(() => {
-        name: product.title,
+          console.log(`Timeout waiting for content on ${productId}`);
-        price: price,
+        });
-        link: product.link,
+
-      });
+      const html = await page.content();
-    } catch {
+      const $ = cheerio.load(html);
-      productsWithBrowser.push(product);
+
+      const items = $(".product-offer__container").first();
+      for (const item of items) {
+        let name = $(item).data("productname");
+        let price = $(item).data("price");
+        let link = `https://www.ceneo.pl/${$(item).data("click-url")}`;
+        const shop = $(item).data("shopurl") || "ceneo.pl";
+
+        if (!name || !price) {
+          name = $(item).find(".short-name__txt").text();
+          price = $(item).find(".price").text();
+          link = `https://www.ceneo.pl/${productId}`;
+        }
+
+        if (!name || !price || !link) {
+          continue;
+        }
+
+        products.push({ name, price, link, shop });
+      }
+    } catch (error) {
+      console.error(`Error fetching product ${productId}:`, error.message);
     }
   }
 
-  if (productsWithBrowser.length > 0) {
+  await browser.close();
-    const browser = await chromium.launch({ headless: false });
-    const context = await browser.newContext({
-      userAgent:
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
-    });
-    const page = await context.newPage();
 
-    for (const product of productsWithBrowser) {
+  await compareAndSave(products);
-      await page.goto(product.link);
+  console.log("Aktualne ceny zapisane w productsPrice.json");
-      const selector = selectors.find((selector) =>
-        product.link.includes(selector)
-      );
-      const element = await page.$(priceSelectors[selector]);
-
-      if (!element) continue;
-
-      const price = await element.textContent();
-      productsPrice.push({
-        name: product.title,
-        price: price,
-        link: product.link,
-      });
-    }
-
-    await browser.close();
-  }
-
-  await compareAndSave(productsPrice);
-
-  console.log("Done! Check productsPrice.json");
 }
 
-cron.schedule("* */3 * * *", async () => {
+sendMessage("Startuję monitoring cen");
-  await init();
+const task = cron.schedule("*/15 7-23 * * *", init, {
+  timezone: "Europe/Warsaw",
 });
+
+cron.schedule("0 7 * * *", () => {
+  const date = new Date().toLocaleDateString("pl-PL");
+  sendMessage(`Zaczynamy monitoring ${date}`);
+});
+
+task.execute();

index.js.old

@@ -0,0 +1,153 @@
+const cheerio = require("cheerio");
+const cron = require("node-cron");
+const axios = require("axios");
+const { chromium } = require("playwright");
+const fs = require("fs");
+
+const discordWebhook =
+  "https://discord.com/api/webhooks/1439286509390921749/t2Hb8XloF6zhDRYD1yh_QlkHHa9eHUyXvd9TxZRHwqR_b_OxxbnwDgsm4em8TwA9NQIa";
+
+const priceSelectors = {
+  miodowamydlarnia: ".projector_prices__price",
+  greentouch: ".main-price",
+  amazon: "#corePrice_feature_div .a-price .a-offscreen",
+  soxo: "#projector_price_value span",
+  empik: '[data-ta-section="priceMainContainer"] [data-ta="price"]',
+  notino: "#pd-price",
+};
+
+const excludePage = ["allegro", "homla.com.pl", "home-you.com"];
+
+function sendMessage(message) {
+  axios.post(discordWebhook, {
+    content: message,
+  });
+}
+
+async function compareAndSave(productsPrice) {
+  if (!fs.existsSync("productsPrice.json")) {
+    fs.writeFileSync("productsPrice.json", "[]");
+  }
+
+  const productsPriceJson = fs.readFileSync("productsPrice.json", "utf8");
+  const oldProductsPrice = JSON.parse(productsPriceJson);
+
+  const diffProducts = [];
+
+  for (const product of productsPrice) {
+    const oldProduct = oldProductsPrice.find(
+      (oldProduct) => oldProduct.link === product.link
+    );
+
+    if (oldProduct && oldProduct.price !== product.price) {
+      diffProducts.push({
+        name: product.name,
+        newPrice: product.price,
+        oldPrice: oldProduct.price,
+        link: product.link,
+      });
+    }
+  }
+
+  for (const product of diffProducts) {
+    sendMessage(
+      `Zmiana ceny **${product.name}**:\nCena: ${product.oldPrice} -> ${product.newPrice}\nLink: ${product.link}`
+    );
+  }
+
+  if (diffProducts.length === 0) {
+    sendMessage("Brak zmian w cenach");
+  }
+
+  fs.writeFileSync(
+    "productsPrice.json",
+    JSON.stringify(productsPrice, null, 2)
+  );
+}
+
+async function getProducts() {
+  const products = await axios
+    .get(
+      "https://db.maciaszek.ovh/api/collections/gifts_items/records?fields=title,link"
+    )
+    .then((response) => response.data.items);
+
+  return products
+    .filter((product) => product.link !== "")
+    .filter(
+      (product) => !excludePage.some((page) => product.link.includes(page))
+    );
+}
+
+async function init() {
+  const productsWithLinks = await getProducts();
+  const productsWithBrowser = [];
+  const productsPrice = [];
+  const selectors = Object.keys(priceSelectors);
+
+  console.log("Zaczynam sprawdzać ceny");
+  for (const product of productsWithLinks) {
+    if (product.link === "") continue;
+
+    try {
+      const { data } = await axios.get(product.link, {
+        headers: {
+          "User-Agent":
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
+        },
+      });
+
+      const $ = cheerio.load(data);
+      const selector = selectors.find((selector) =>
+        product.link.includes(selector)
+      );
+      const price = $(priceSelectors[selector]).text();
+      productsPrice.push({
+        name: product.title,
+        price: price,
+        link: product.link,
+      });
+    } catch {
+      productsWithBrowser.push(product);
+    }
+  }
+
+  if (productsWithBrowser.length > 0) {
+    const browser = await chromium.launch();
+    const context = await browser.newContext({
+      userAgent:
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
+    });
+    const page = await context.newPage();
+
+    for (const product of productsWithBrowser) {
+      await page.goto(product.link);
+      const selector = selectors.find((selector) =>
+        product.link.includes(selector)
+      );
+      const element = await page.$(priceSelectors[selector]);
+
+      if (!element) continue;
+
+      const price = await element.textContent();
+      productsPrice.push({
+        name: product.title,
+        price: price,
+        link: product.link,
+      });
+    }
+
+    await browser.close();
+  }
+
+  await compareAndSave(productsPrice);
+
+  console.log("Sprawdzone! Aktualne ceny zapisane w productsPrice.json");
+}
+
+sendMessage("Zaczynam monitoring cen");
+const task = cron.schedule("0 6,9,12,15,18,21 * * *", init, {
+  timezone: "Europe/Warsaw",
+});
+
+task.execute();