{
  "_id": "6ad10e800edca5be250217f1",
  "Package": "robotstxtr",
  "Type": "Package",
  "Title": "Engine-Aware 'robots.txt' Policy with Google's Matcher Algorithm",
  "Version": "0.3.1",
  "Language": "en-US",
  "Authors@R": "c(person(\"Bart\", \"Turczynski\", email = \"bartek@turczynski.pl\", role = c(\"aut\", \"cre\", \"cph\"),\ncomment = c(ORCID = \"0000-0002-8788-7980\")),\nperson(\"Google LLC\", role = \"cph\",\ncomment = \"Bundled 'robots.txt' matcher: src/robots.cc, src/robots.h, src/reporting_robots.cc, src/reporting_robots.h\"))",
  "Description": "Evaluates whether URLs may be crawled according to\n'robots.txt' rules. A versioned engine-aware facade keeps\nneutral fetch evidence, per-engine status policy, robots\nproduct tokens, and matcher capabilities separate, returning\nexplicit non-decisions when policy is undocumented or a matcher\nis unavailable. The available matcher is a bundled,\nupstream-test-validated snapshot of Google's open-source C++\n'robots.txt' matcher, preserving 'Googlebot' behavior. Fetches\nuse a deterministic, conservative policy built on 'httr2', with\nfetch origins constructed by 'rurl'. All network access in\ntests, examples, and vignettes is mocked or served locally, so\nno internet connection is required to install, test, or check\nthe package. This matcher derives from Google's 'robotstxt'\nproject <https://github.com/google/robotstxt>, distributed\nunder the Apache License 2.0; see the package NOTICE.",
  "License": "MIT + file LICENSE",
  "Copyright": "file inst/COPYRIGHTS",
  "URL": "https://bart-turczynski.gitlab.io/robotstxtr/,\nhttps://gitlab.com/bart-turczynski/robotstxtr,\nhttps://bart-turczynski.r-universe.dev/robotstxtr",
  "BugReports": "https://gitlab.com/bart-turczynski/robotstxtr/-/issues",
  "Encoding": "UTF-8",
  "Roxygen": "list(markdown = TRUE)",
  "SystemRequirements": "C++17",
  "Config/testthat/edition": "3",
  "VignetteBuilder": "knitr",
  "Config/roxygen2/version": "8.0.0",
  "Config/pak/sysreqs": "libicu-dev libssl-dev",
  "Repository": "https://bart-turczynski.r-universe.dev",
  "Date/Publication": "2026-10-15 14:12:52 UTC",
  "RemoteUrl": "https://gitlab.com/bart-turczynski/robotstxtr",
  "RemoteRef": "HEAD",
  "RemoteSha": "da6da01207733a724ee200b7f085c56c3159b1e9",
  "NeedsCompilation": "yes",
  "Packaged": {
    "Date": "2026-10-15 17:18:56 UTC",
    "User": "root"
  },
  "Author": "Bart Turczynski [aut, cre, cph] (ORCID:\n<https://orcid.org/0000-0002-8788-7980>),\nGoogle LLC [cph] (Bundled 'robots.txt' matcher: src/robots.cc,\nsrc/robots.h, src/reporting_robots.cc, src/reporting_robots.h)",
  "Maintainer": "Bart Turczynski <bartek@turczynski.pl>",
  "_user": "bart-turczynski",
  "_type": "src",
  "_file": "robotstxtr_0.3.1.tar.gz",
  "_fileid": "https://r2.ropensci.org/a9a76af8cc312e6c40b18504ec93d8ebd1bfeedc2dac2a22d01ca9af9cd09104",
  "_filesize": 490566,
  "_sha256": "a9a76af8cc312e6c40b18504ec93d8ebd1bfeedc2dac2a22d01ca9af9cd09104",
  "_expires": "2027-01-23T17:33:47.000Z",
  "_created": "2026-10-15T17:18:56.000Z",
  "_published": "2026-10-15T17:33:52.780Z",
  "_buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898",
  "_status": "success",
  "_host": "GitHub-Actions",
  "_jobs": [
    {
      "job": 116564995338,
      "time": 144,
      "config": "linux-devel-arm64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "11993597631"
    },
    {
      "job": 116564995510,
      "time": 190,
      "config": "linux-devel-x86_64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "11993717532"
    },
    {
      "job": 116564995325,
      "time": 177,
      "config": "linux-release-arm64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "11994605940"
    },
    {
      "job": 116564995282,
      "time": 162,
      "config": "linux-release-x86_64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "11993154069"
    },
    {
      "job": 116564995344,
      "time": 154,
      "config": "macos-oldrel-arm64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "11993079439"
    },
    {
      "job": 116564995759,
      "time": 294,
      "config": "macos-oldrel-x86_64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "11993294034"
    },
    {
      "job": 116564995302,
      "time": 166,
      "config": "macos-release-arm64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "11994716093"
    },
    {
      "job": 116564995658,
      "time": 415,
      "config": "macos-release-x86_64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "11993938591"
    },
    {
      "job": 116562824516,
      "time": 265,
      "config": "source",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "11993692081"
    },
    {
      "job": 116564995125,
      "time": 128,
      "config": "wasm-release",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "11994845169"
    },
    {
      "job": 116564995450,
      "time": 359,
      "config": "windows-devel-arm64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "11994451471"
    },
    {
      "job": 116564995517,
      "time": 236,
      "config": "windows-devel-x86_64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "11994516184"
    },
    {
      "job": 116564995594,
      "time": 193,
      "config": "windows-oldrel-x86_64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "11994820387"
    },
    {
      "job": 116564995616,
      "time": 298,
      "config": "windows-release-arm64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "11993219474"
    },
    {
      "job": 116564995528,
      "time": 214,
      "config": "windows-release-x86_64",
      "r": "4.6.1",
      "check": "OK",
      "artifact": "11994421189"
    }
  ],
  "_upstream": "https://gitlab.com/bart-turczynski/robotstxtr",
  "_commit": {
    "id": "da6da01207733a724ee200b7f085c56c3159b1e9",
    "author": "Bart Turczynski <bartek@turczynski.pl>",
    "committer": "Bart Turczynski <bartek@turczynski.pl>",
    "message": "Merge branch 'chore/record-cran-submission-0.3.1' into 'main'\n\nchore: record the 0.3.1 CRAN submission\n\nSee merge request bart-turczynski/robotstxtr!62",
    "time": 1792073572
  },
  "_maintainer": {
    "name": "Bart Turczynski",
    "email": "bartek@turczynski.pl",
    "login": "bart-turczynski",
    "linkedin": "in/bart-turczynski",
    "orcid": "0000-0002-8788-7980",
    "twitter": "@bart_turczynski",
    "description": "",
    "uuid": 142225707
  },
  "_distro": "resolute",
  "_registered": true,
  "_dependencies": [
    {
      "package": "R",
      "version": ">= 4.1.0",
      "role": "Depends"
    },
    {
      "package": "cpp11",
      "role": "LinkingTo"
    },
    {
      "package": "httr2",
      "version": ">= 1.2.0",
      "role": "Imports"
    },
    {
      "package": "rurl",
      "version": ">= 3.0.1",
      "role": "Imports"
    },
    {
      "package": "callr",
      "role": "Suggests"
    },
    {
      "package": "httpuv",
      "role": "Suggests"
    },
    {
      "package": "jsonlite",
      "role": "Suggests"
    },
    {
      "package": "knitr",
      "role": "Suggests"
    },
    {
      "package": "oysteR",
      "role": "Suggests"
    },
    {
      "package": "rmarkdown",
      "role": "Suggests"
    },
    {
      "package": "rosv",
      "role": "Suggests"
    },
    {
      "package": "testthat",
      "version": ">= 3.0.0",
      "role": "Suggests"
    }
  ],
  "_owner": "gitlab-bart-turczynski",
  "_selfowned": true,
  "_usedby": 0,
  "_updates": [
    {
      "week": "2026-28",
      "n": 20
    },
    {
      "week": "2026-29",
      "n": 28
    },
    {
      "week": "2026-30",
      "n": 18
    },
    {
      "week": "2026-31",
      "n": 7
    },
    {
      "week": "2026-35",
      "n": 2
    },
    {
      "week": "2026-37",
      "n": 7
    },
    {
      "week": "2026-39",
      "n": 21
    },
    {
      "week": "2026-40",
      "n": 22
    },
    {
      "week": "2026-41",
      "n": 5
    },
    {
      "week": "2026-42",
      "n": 6
    }
  ],
  "_tags": [
    {
      "name": "v0.1.0",
      "date": "2026-07-15"
    },
    {
      "name": "v0.2.0",
      "date": "2026-07-19"
    },
    {
      "name": "v0.3.0",
      "date": "2026-09-28"
    }
  ],
  "_userbio": {
    "uuid": 142225707,
    "type": "user",
    "name": "Bart Turczynski",
    "followers": 4
  },
  "_downloads": {
    "count": 0,
    "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/robotstxtr"
  },
  "_devurl": "https://gitlab.com/bart-turczynski/robotstxtr",
  "_pkgdown": "https://bart-turczynski.gitlab.io/robotstxtr/",
  "_searchresults": 6,
  "_topics": [
    "cpp"
  ],
  "_rbuild": "4.6.1",
  "_assets": [
    "extra/citation.cff",
    "extra/citation.html",
    "extra/citation.json",
    "extra/citation.txt",
    "extra/contents.json",
    "extra/manual.pdf",
    "extra/NEWS.html",
    "extra/NEWS.txt",
    "extra/readme.html",
    "extra/readme.md",
    "extra/robotstxtr.html",
    "LICENSE"
  ],
  "_cranurl": false,
  "_exports": [
    "allowed_by_robots_text",
    "allowed_by_robots_url",
    "as_legacy_robots_decisions_v1",
    "robots_body",
    "robots_engine_contract_v1",
    "robots_engine_contract_v2",
    "robots_evaluate_text_v1",
    "robots_evaluate_url_v1",
    "robots_fetch",
    "robots_resolve_matcher_profile_v1",
    "robots_validate_text",
    "robots_validate_url"
  ],
  "_help": [
    {
      "page": "allowed_by_robots_text",
      "title": "Match URLs against a supplied robots.txt body",
      "topics": [
        "allowed_by_robots_text"
      ]
    },
    {
      "page": "allowed_by_robots_url",
      "title": "Fetch robots.txt and match URLs under the deterministic fetch policy",
      "topics": [
        "allowed_by_robots_url"
      ]
    },
    {
      "page": "as_legacy_robots_decisions_v1",
      "title": "Convert a v1 Google engine result to the legacy decision schema",
      "topics": [
        "as_legacy_robots_decisions_v1"
      ]
    },
    {
      "page": "print.robots_decisions",
      "title": "Print a 'robots_decisions' object",
      "topics": [
        "print.robots_decisions"
      ]
    },
    {
      "page": "print.robots_fetches",
      "title": "Print a 'robots_fetches' object",
      "topics": [
        "print.robots_fetches"
      ]
    },
    {
      "page": "print.robots_validations",
      "title": "Print a robots.txt validation result",
      "topics": [
        "print.robots_validations"
      ]
    },
    {
      "page": "robots_body",
      "title": "Preview or extract a stored robots.txt body",
      "topics": [
        "robots_body"
      ]
    },
    {
      "page": "robots_engine_contract_v1",
      "title": "Inspect the versioned engine-aware robots contract",
      "topics": [
        "robots_engine_contract_v1"
      ]
    },
    {
      "page": "robots_engine_contract_v2",
      "title": "Inspect the engine-aware/v2 robots contract (Bing activation)",
      "topics": [
        "robots_engine_contract_v2"
      ]
    },
    {
      "page": "robots_evaluate_text_v1",
      "title": "Evaluate URLs against supplied robots.txt under an explicit engine profile",
      "topics": [
        "robots_evaluate_text_v1"
      ]
    },
    {
      "page": "robots_evaluate_url_v1",
      "title": "Fetch and evaluate robots.txt under an explicit engine profile",
      "topics": [
        "robots_evaluate_url_v1"
      ]
    },
    {
      "page": "robots_fetch",
      "title": "Fetch robots.txt documents for URLs under the deterministic fetch policy",
      "topics": [
        "robots_fetch"
      ]
    },
    {
      "page": "robots_resolve_matcher_profile_v1",
      "title": "Resolve a matcher-profile selector before evaluating anything",
      "topics": [
        "robots_resolve_matcher_profile_v1"
      ]
    },
    {
      "page": "robots_validate_text",
      "title": "Validate a supplied robots.txt document",
      "topics": [
        "robots_validate_text"
      ]
    },
    {
      "page": "robots_validate_url",
      "title": "Fetch and validate robots.txt documents for URLs",
      "topics": [
        "robots_validate_url"
      ]
    }
  ],
  "_readme": "https://gitlab.com/bart-turczynski/robotstxtr/raw/HEAD/README.md",
  "_rundeps": [
    "askpass",
    "cli",
    "cpp11",
    "curl",
    "digest",
    "glue",
    "httr2",
    "lifecycle",
    "magrittr",
    "openssl",
    "pslr",
    "punycoder",
    "R6",
    "raddr",
    "Rcpp",
    "rlang",
    "rurl",
    "stringi",
    "sys",
    "vctrs",
    "withr"
  ],
  "_sysdeps": [
    {
      "shlib": "libstdc++",
      "package": "libstdc++6",
      "source": "gcc",
      "version": "16-20260322-1ubuntu1",
      "name": "c++",
      "homepage": "http://gcc.gnu.org/",
      "description": "GNU Standard C++ Library v3"
    }
  ],
  "_vignettes": [
    {
      "source": "introduction.Rmd",
      "filename": "introduction.html",
      "title": "Introduction to robotstxtr",
      "engine": "knitr::rmarkdown",
      "headings": [
        "What robotstxtr does",
        "Crawl-decision entry points",
        "Matching against a supplied body",
        "Vectorized, with per-URL user agents",
        "Rich match metadata",
        "Inspecting the stored body",
        "Fetching robots.txt for a URL",
        "Validating the document itself",
        "The URL you pass is the URL that is matched",
        "Summary"
      ],
      "created": "2026-07-10 18:59:44",
      "modified": "2026-07-18 00:44:32",
      "commits": 4
    }
  ],
  "_score": 3.6989700043360187,
  "_indexed": true,
  "_nocasepkg": "robotstxtr",
  "_universes": [
    "bart-turczynski",
    "gitlab-bart-turczynski"
  ],
  "_binaries": [
    {
      "r": "4.7.0",
      "os": "linux",
      "version": "0.3.1",
      "date": "2026-10-15T17:21:00.000Z",
      "distro": "resolute",
      "arch": "aarch64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/81d160795c59a442c3accff53f8b2e30af59605019574c7b9b341bf683b6ddbc",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.7.0",
      "os": "linux",
      "version": "0.3.1",
      "date": "2026-10-15T17:21:38.000Z",
      "distro": "resolute",
      "arch": "x86_64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/04f47ebb488804ed22cb942f137659b2db6bdaa6dd847cf86c45993210b25e31",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.6.1",
      "os": "linux",
      "version": "0.3.1",
      "date": "2026-10-15T17:21:36.000Z",
      "distro": "resolute",
      "arch": "aarch64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/dad8845b1a9ad67b54b39d04af2ff19c5cd1897b597a7ab993c24d4257ccc726",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.6.1",
      "os": "linux",
      "version": "0.3.1",
      "date": "2026-10-15T17:21:23.000Z",
      "distro": "resolute",
      "arch": "x86_64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/b6dc0c37d8c8a5cf77ea70b8b1e7640196d0acb332dc74d3a4c20c5264b5c664",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.5.3",
      "os": "mac",
      "version": "0.3.1",
      "date": "2026-10-15T17:21:16.000Z",
      "arch": "aarch64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/3983c4281df8c867191cf43cddaede0fe9cbfaa5949be6754c24b55f6c718fc7",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.5.3",
      "os": "mac",
      "version": "0.3.1",
      "date": "2026-10-15T17:26:16.000Z",
      "arch": "x86_64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/0028b8193d28f15806b8fe2037007930d8149d3b7d7ab2d52d8d48adc8dabc1a",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.6.1",
      "os": "mac",
      "version": "0.3.1",
      "date": "2026-10-15T17:23:10.000Z",
      "arch": "aarch64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/0df656b43c91b5fe11684271259f8a7796a24b79acad65df06108309f05d4dcc",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.6.1",
      "os": "mac",
      "version": "0.3.1",
      "date": "2026-10-15T17:30:13.000Z",
      "arch": "x86_64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/590cec9206a092409592592f6d0428de70aa70dd11503e00acc69a1df32959c2",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.6.0",
      "os": "wasm",
      "version": "0.3.1",
      "date": "2026-10-15T17:21:11.000Z",
      "arch": "emscripten",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/b76087d67f0756ee7ffaee078748019120f5ea24638fefdcf55054ee873f6a58",
      "status": "success",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.7.0",
      "os": "win",
      "version": "0.3.1",
      "date": "2026-10-15T17:23:22.000Z",
      "arch": "aarch64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/323ff522bb59fce96c0b685f33a291b64865a3cfaebf47e613093914dcd4cecb",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.7.0",
      "os": "win",
      "version": "0.3.1",
      "date": "2026-10-15T17:20:34.000Z",
      "arch": "x86_64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/74ecf6a822f5c74ba203f2990ce3d4a0ed9024bb77d76ebe1240493108972f19",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.5.3",
      "os": "win",
      "version": "0.3.1",
      "date": "2026-10-15T17:20:34.000Z",
      "arch": "x86_64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/f788cc53bad44841f5611f084c6b18df3152f1c9086679d1cfa94183fdc48dc1",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.6.1",
      "os": "win",
      "version": "0.3.1",
      "date": "2026-10-15T17:22:21.000Z",
      "arch": "aarch64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/f6680fe6fbadbc12efeebb21d0ce0f8df1a94988c2e3bd0dc1b1fbf5bc6b0fe8",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    },
    {
      "r": "4.6.1",
      "os": "win",
      "version": "0.3.1",
      "date": "2026-10-15T17:20:36.000Z",
      "arch": "x86_64",
      "commit": "da6da01207733a724ee200b7f085c56c3159b1e9",
      "fileid": "https://r2.ropensci.org/81151ecc600ccd761a29aba72f9f009add53320830e8d8142f6986d9c987367a",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/bart-turczynski/actions/runs/38775701898"
    }
  ]
}