User-agent: * Allow: / Disallow: /*/dashboard/ Disallow: /*/tools/*?* # Strona powrotu z platnosci - niesie identyfikator sesji konkretnego kupujacego. Disallow: /*/price-database/thank-you Disallow: /badge/ Disallow: /chart/ # Shardy datasetow zostaja zamkniete - nie ma powodu, zeby crawler ciagnal # gigabajty gzipa. Ale manifest i latest.json MUSZA byc osiagalne: to na # nie wskazuje contentUrl w danych strukturalnych Dataset, a Google Dataset # Search bez nich nie potwierdzi, ze dystrybucja istnieje. # Same strony opisu siedza pod /{lang}/datasets i nie sa tu blokowane. Allow: /datasets/releases/*/latest.json Allow: /datasets/releases/*/*/manifest.json Disallow: /datasets/ Disallow: /dashboard Disallow: /login Disallow: /register # Boty AI dopuszczone jawnie, ale KAZDA grupa powtarza komplet zakazow. # Standard robots.txt mowi, ze crawler stosuje wylacznie najbardziej pasujaca # grupe - regul z "User-agent: *" NIE dokleja. Wczesniej nazwane grupy mialy # samo "Allow: /", wiec /dashboard/, /login i strona powrotu z platnosci # (niosaca identyfikator sesji kupujacego) byly dla tych botow otwarte. # # Linki partnerskie /offer/redirect/ sa CELOWO dozwolone. Za agentem # wywolanym przez czlowieka (ChatGPT-User, Claude-User) stoi konkretna # osoba z intencja zakupowa, a to sa linki zarobkowe - blokowanie ich # odcinaloby realny przychod. Kliknieciom robota nadajemy etykiete po # stronie serwera, wiec zostaja policzone i nadal odrozniale. User-agent: GPTBot Allow: / Disallow: /*/dashboard/ Disallow: /*/tools/*?* # Strona powrotu z platnosci - niesie identyfikator sesji konkretnego kupujacego. Disallow: /*/price-database/thank-you Disallow: /badge/ Disallow: /chart/ Allow: /datasets/releases/*/latest.json Allow: /datasets/releases/*/*/manifest.json Disallow: /datasets/ Disallow: /dashboard Disallow: /login Disallow: /register User-agent: OAI-SearchBot Allow: / Disallow: /*/dashboard/ Disallow: /*/tools/*?* # Strona powrotu z platnosci - niesie identyfikator sesji konkretnego kupujacego. Disallow: /*/price-database/thank-you Disallow: /badge/ Disallow: /chart/ Allow: /datasets/releases/*/latest.json Allow: /datasets/releases/*/*/manifest.json Disallow: /datasets/ Disallow: /dashboard Disallow: /login Disallow: /register User-agent: ChatGPT-User Allow: / Disallow: /*/dashboard/ Disallow: /*/tools/*?* # Strona powrotu z platnosci - niesie identyfikator sesji konkretnego kupujacego. Disallow: /*/price-database/thank-you Disallow: /badge/ Disallow: /chart/ Allow: /datasets/releases/*/latest.json Allow: /datasets/releases/*/*/manifest.json Disallow: /datasets/ Disallow: /dashboard Disallow: /login Disallow: /register User-agent: ClaudeBot Allow: / Disallow: /*/dashboard/ Disallow: /*/tools/*?* # Strona powrotu z platnosci - niesie identyfikator sesji konkretnego kupujacego. Disallow: /*/price-database/thank-you Disallow: /badge/ Disallow: /chart/ Allow: /datasets/releases/*/latest.json Allow: /datasets/releases/*/*/manifest.json Disallow: /datasets/ Disallow: /dashboard Disallow: /login Disallow: /register User-agent: Claude-User Allow: / Disallow: /*/dashboard/ Disallow: /*/tools/*?* # Strona powrotu z platnosci - niesie identyfikator sesji konkretnego kupujacego. Disallow: /*/price-database/thank-you Disallow: /badge/ Disallow: /chart/ Allow: /datasets/releases/*/latest.json Allow: /datasets/releases/*/*/manifest.json Disallow: /datasets/ Disallow: /dashboard Disallow: /login Disallow: /register User-agent: Claude-SearchBot Allow: / Disallow: /*/dashboard/ Disallow: /*/tools/*?* # Strona powrotu z platnosci - niesie identyfikator sesji konkretnego kupujacego. Disallow: /*/price-database/thank-you Disallow: /badge/ Disallow: /chart/ Allow: /datasets/releases/*/latest.json Allow: /datasets/releases/*/*/manifest.json Disallow: /datasets/ Disallow: /dashboard Disallow: /login Disallow: /register User-agent: PerplexityBot Allow: / Disallow: /*/dashboard/ Disallow: /*/tools/*?* # Strona powrotu z platnosci - niesie identyfikator sesji konkretnego kupujacego. Disallow: /*/price-database/thank-you Disallow: /badge/ Disallow: /chart/ Allow: /datasets/releases/*/latest.json Allow: /datasets/releases/*/*/manifest.json Disallow: /datasets/ Disallow: /dashboard Disallow: /login Disallow: /register User-agent: Google-Extended Allow: / Disallow: /*/dashboard/ Disallow: /*/tools/*?* # Strona powrotu z platnosci - niesie identyfikator sesji konkretnego kupujacego. Disallow: /*/price-database/thank-you Disallow: /badge/ Disallow: /chart/ Allow: /datasets/releases/*/latest.json Allow: /datasets/releases/*/*/manifest.json Disallow: /datasets/ Disallow: /dashboard Disallow: /login Disallow: /register User-agent: Applebot-Extended Allow: / Disallow: /*/dashboard/ Disallow: /*/tools/*?* # Strona powrotu z platnosci - niesie identyfikator sesji konkretnego kupujacego. Disallow: /*/price-database/thank-you Disallow: /badge/ Disallow: /chart/ Allow: /datasets/releases/*/latest.json Allow: /datasets/releases/*/*/manifest.json Disallow: /datasets/ Disallow: /dashboard Disallow: /login Disallow: /register User-agent: CCBot Allow: / Disallow: /*/dashboard/ Disallow: /*/tools/*?* # Strona powrotu z platnosci - niesie identyfikator sesji konkretnego kupujacego. Disallow: /*/price-database/thank-you Disallow: /badge/ Disallow: /chart/ Allow: /datasets/releases/*/latest.json Allow: /datasets/releases/*/*/manifest.json Disallow: /datasets/ Disallow: /dashboard Disallow: /login Disallow: /register # Dokumentacja dla agentow AI: # API https://pricelists.org/en/agent-docs # OpenAPI https://pricelists.org/.well-known/openapi.json # LLM https://pricelists.org/llms.txt Sitemap: https://pricelists.org/pl/sitemaps/pages-index.xml Sitemap: https://pricelists.org/en/sitemaps/pages-index.xml # Kohorta priorytetowa - najlepsze produkty z realnym porownaniem cen. # Zastapila zgloszenie calej tabeli items (228 657 765 adresow na jezyk): # domena crawlowana w tempie 298 adresow na dobe nie ma jak tego przejsc. Sitemap: https://pricelists.org/pl/sitemaps/products-a-index.xml Sitemap: https://pricelists.org/en/sitemaps/products-a-index.xml Sitemap: https://pricelists.org/pl/sitemaps/entities-index.xml Sitemap: https://pricelists.org/en/sitemaps/entities-index.xml Sitemap: https://pricelists.org/pl/sitemaps/brands-index.xml Sitemap: https://pricelists.org/en/sitemaps/brands-index.xml Sitemap: https://pricelists.org/pl/sitemaps/categories-index.xml Sitemap: https://pricelists.org/en/sitemaps/categories-index.xml # Zlokalizowane strony statyczne (UI przetłumaczone na 21 dodatkowych języków) Sitemap: https://pricelists.org/de/sitemaps/pages-index.xml Sitemap: https://pricelists.org/fr/sitemaps/pages-index.xml Sitemap: https://pricelists.org/es/sitemaps/pages-index.xml Sitemap: https://pricelists.org/it/sitemaps/pages-index.xml Sitemap: https://pricelists.org/pt/sitemaps/pages-index.xml Sitemap: https://pricelists.org/nl/sitemaps/pages-index.xml Sitemap: https://pricelists.org/cs/sitemaps/pages-index.xml Sitemap: https://pricelists.org/sk/sitemaps/pages-index.xml Sitemap: https://pricelists.org/ro/sitemaps/pages-index.xml Sitemap: https://pricelists.org/uk/sitemaps/pages-index.xml Sitemap: https://pricelists.org/ru/sitemaps/pages-index.xml Sitemap: https://pricelists.org/sv/sitemaps/pages-index.xml Sitemap: https://pricelists.org/tr/sitemaps/pages-index.xml Sitemap: https://pricelists.org/zh/sitemaps/pages-index.xml Sitemap: https://pricelists.org/ja/sitemaps/pages-index.xml Sitemap: https://pricelists.org/ar/sitemaps/pages-index.xml Sitemap: https://pricelists.org/hi/sitemaps/pages-index.xml Sitemap: https://pricelists.org/ko/sitemaps/pages-index.xml Sitemap: https://pricelists.org/id/sitemaps/pages-index.xml Sitemap: https://pricelists.org/vi/sitemaps/pages-index.xml Sitemap: https://pricelists.org/th/sitemaps/pages-index.xml