You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
34 lines
997 B
34 lines
997 B
% Generated by roxygen2: do not edit by hand
|
|
% Please edit documentation in R/can-fetch.r
|
|
\name{can_fetch}
|
|
\alias{can_fetch}
|
|
\title{Test URL paths against a \code{robxp} \code{robots.txt} object}
|
|
\usage{
|
|
can_fetch(obj, path = "/", user_agent = "*")
|
|
}
|
|
\arguments{
|
|
\item{obj}{\code{robxp} object}
|
|
|
|
\item{path}{path to test}
|
|
|
|
\item{user_agent}{user agent to test}
|
|
}
|
|
\value{
|
|
logical vector indicating whether you have permission to fetch the content
|
|
}
|
|
\description{
|
|
Provide a character vector of URL paths plus optional user agent and this function will
|
|
return a logical vector indicating whether you have permission to fetch the content
|
|
at the respective path.
|
|
}
|
|
\examples{
|
|
gh <- paste0(readLines(system.file("extdata", "github-robots.txt",
|
|
package="spiderbar")), collapse="\n")
|
|
gh_rt <- robxp(gh)
|
|
|
|
can_fetch(gh_rt, "/humans.txt", "*") # TRUE
|
|
can_fetch(gh_rt, "/login", "*") # FALSE
|
|
can_fetch(gh_rt, "/oembed", "CCBot") # FALSE
|
|
|
|
can_fetch(gh_rt, c("/humans.txt", "/login", "/oembed"))
|
|
}
|
|
|