# robots.txt for Magento 1.9.x / v1.6 2018-08-19 / Peeter Marvet # (original version from 2015, edited in 2017 to add filter query parameter disallow samples + some wildcards, # edited in 2018 to add query params blocking to Yandex as named User-agent does not read *) # based on: # http://inchoo.net/ecommerce/ultimate-magento-robots-txt-file-examples/ # http://www.byte.nl/blog/magento-robots-txt/ # https://astrio.net/blog/optimize-robots-txt-for-magento/ # # comment and clone at https://gist.github.com/petskratt/016c9dbf159a81b9d6aa # Keep in mind that by standard robots.txt should NOT contain empty lines, except between UA blocks! # # Sitemap (uncomment, change and add language/shop specific sitemaps, if running on multiple domains # keep in mind sitemap can only point to own domain so something like sitemapindex.php is needed) Sitemap: https://www.jqueenwatchwinders.de/sitemap_de.xml Sitemap: https://www.jqueenwatchwinders.fr/sitemap_fr.xml # # Crawlers Setup User-agent: * # # Allow paging (unless paging inside a listing with more params, as disallowed below) Allow: /*?p= # # Directories Disallow: /404/ Disallow: /app/ Disallow: /cgi-bin/ Disallow: /downloader/ Disallow: /errors/ Disallow: /includes/ Disallow: /magento/ #Disallow: /media/ Disallow: /media/captcha/ #Disallow: /media/catalog/ Disallow: /media/customer/ Disallow: /media/dhl/ Disallow: /media/downloadable/ Disallow: /media/import/ Disallow: /media/pdf/ Disallow: /media/sales/ Disallow: /media/tmp/ #Disallow: /media/wysiwyg/ Disallow: /media/xmlconnect/ Disallow: /pkginfo/ Disallow: /report/ Disallow: /scripts/ Disallow: /shell/ #Disallow: /skin/ Disallow: /stats/ Disallow: /var/ Disallow: /email/ # # Paths (if using shop id in URL must prefix with * or copy for each) Disallow: */index.php/ Disallow: /index.php/ Disallow: */catalog/product_compare/ Disallow: /catalog/product_compare/ Disallow: */catalog/category/view/ Disallow: /catalog/category/view/ Disallow: */catalog/product/view/ Disallow: /catalog/product/view/ Disallow: */catalog/product/gallery/ Disallow: /catalog/product/gallery/ Disallow: */catalogsearch/ Disallow: /catalogsearch/ Disallow: */control/ Disallow: /control/ Disallow: */contacts/ Disallow: /contacts/ Disallow: */customer/ Disallow: /customer/ Disallow: */customer/account/login/ Disallow: */customize/ Disallow: /customize/ Disallow: */newsletter/ Disallow: /newsletter/ Disallow: */poll/ Disallow: /poll/ Disallow: */review/ Disallow: /review/ Disallow: */sendfriend/ Disallow: /sendfriend/ Disallow: */tag/ Disallow: /tag/ Disallow: */wishlist/ Disallow: /wishlist/ Disallow: */checkout/ Disallow: /checkout/ Disallow: */onestepcheckout/ Disallow: /onestepcheckout/ Disallow: */onepage/ Disallow: /onepage/ Disallow: */intellectual-property-rights/ Disallow: /intellectual-property-rights/ Disallow: */rewardpointsbehavior/index/tweet/ Disallow: /rewardpointsbehavior/index/tweet/ Disallow: */wishlist/ Disallow: /wishlist/ Disallow: */reviews/ Disallow: /reviews/ Disallow: */order/ Disallow: /order/ Disallow: */calendar/* Disallow: */directory/* Disallow: */translate/* Disallow: */advancedreviews/* Allow: */js/lovevox/advancedreviews/ Disallow: */checkbyproscons/* Disallow: */centinel/* Disallow: */paypal/express/* Disallow: */varien/payment_lv Disallow: */varien/form/ Disallow: */varien/opcheckout_ADV Disallow: */varien/bundle Disallow: */varien/configurable/ Disallow: */varien/msrp/ Disallow: */varien/scripts Disallow: /advancedreviews/proscons/checkbyproscons/* Disallow: /amxsearchfront/ Disallow: */mage/centinel Disallow: */mage/translate # # Files Disallow: /cron.php Disallow: /cron.sh Disallow: /error_log Disallow: /install.php Disallow: /LICENSE.html Disallow: /LICENSE.txt Disallow: /LICENSE_AFL.txt Disallow: /STATUS.txt Disallow: .js # # Do not crawl sub category pages that are sorted or filtered. # This would be very broad, could hurt (incl. SEO). # Disallow: /*?* # # These are more specific, pick what you need - and do not forget to add your custom filters! Disallow: /*?dir* Disallow: /*?dir=* Disallow: *?dir=* Disallow: /?dir=* Disallow: /*?limit* Disallow: /*?mode* Disallow: /*?___from_store=* Disallow: /*?___store=* Disallow: /*?cat=* Disallow: /*?q=* Disallow: /*?price=* Disallow: /*?bestseller* Disallow: /*?availability=* Disallow: /*?brand=* Disallow: /*?price* Disallow: /*?filter/price* Disallow: /*?most_viewed* Disallow: /*?reviews_count* Disallow: /*?rating_summary* Disallow: /*filter Disallow: /filter Disallow: /*filter* Disallow: /method=* Disallow: /*?method=* Disallow: /order/tracking/* Disallow: /order/tracking/ Disallow: /catalogsearch/result/* Disallow: /advancedreviews/proscons/* # # Paths that can be safely ignored (no clean URLs) #Disallow: /*?p=*& Disallow: /*.php$ Disallow: /*?SID= # # User-Agent:Pinterest/0.2 (+http://www.pinterest.com/) Allow:/ User-Agent: almaden Disallow: / User-Agent: ASPSeek Disallow: / User-Agent: Axmo Disallow: / User-Agent: BaiduSpider Disallow: / User-Agent: booch Disallow: / User-Agent: DTS Agent Disallow: / User-Agent: Downloader Disallow: / User-Agent: EmailCollector Disallow: / User-Agent: EmailSiphon Disallow: / User-Agent: EmailWolf Disallow: / User-Agent: Expired Domain Sleuth Disallow: / User-Agent: Franklin Locator Disallow: / User-Agent: Gaisbot Disallow: / User-Agent: grub Disallow: / User-Agent: HughCrawler Disallow: / User-Agent: iaea.org Disallow: / User-Agent: lcabotAccept Disallow: / User-Agent: IconSurf Disallow: / User-Agent: Iltrovatore-Setaccio Disallow: / User-Agent: Indy Library Disallow: / User-Agent: IUPUI Disallow: / User-Agent: Kittiecentral Disallow: / User-Agent: larbin Disallow: / User-Agent: lwp-trivial Disallow: / User-Agent: MetaTagRobot Disallow: / User-Agent: Missigua Locator Disallow: / User-Agent: NetResearchServer Disallow: / User-Agent: NextGenSearch Disallow: / User-Agent: NPbot Disallow: / User-Agent: Nutch Disallow: / User-Agent: ObjectsSearch Disallow: / User-Agent: Oracle Ultra Search Disallow: / User-Agent: PEERbot Disallow: / User-Agent: PictureOfInternet Disallow: / User-Agent: PlantyNet Disallow: / User-Agent: QuepasaCreep Disallow: / User-Agent: ScSpider Disallow: / User-Agent: SOFT411 Disallow: / User-Agent: spider.acont.de Disallow: / User-Agent: Sqworm Disallow: / User-Agent: SSM Agent Disallow: / User-Agent: TAMU Disallow: / User-Agent: TheUsefulbot Disallow: / User-Agent: TurnitinBot Disallow: / User-Agent: Tutorial Crawler Disallow: / User-Agent: TutorGig Disallow: / User-Agent: WebCopier Disallow: / User-Agent: WebZIP Disallow: / User-Agent: ZipppBot Disallow: / User-Agent: Xenu Disallow: / User-Agent: Wotbox Disallow: / User-Agent: Wget Disallow: / User-Agent: NaverBot Disallow: / User-Agent: mozDex Disallow: / User-Agent: Sosospider Disallow: / User-agent: Sogou web spider Disallow: / User-agent: sogou spider Disallow: / User-agent: BLEXBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: YisouSpider Disallow: / User-agent: ZoominfoBot Disallow: / User-agent: PetalBot Disallow: / User-agent: YandexBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: yisouspider Disallow: / User-agent: coccocbot Disallow: / User-agent: BLEXBot Disallow: /