You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
23 lines
510 B
23 lines
510 B
% Generated by roxygen2: do not edit by hand
|
|
% Please edit documentation in R/rep.r
|
|
\name{can_fetch}
|
|
\alias{can_fetch}
|
|
\title{Test URL path against robots.txt}
|
|
\usage{
|
|
can_fetch(obj, path = "/", user_agent = "*")
|
|
}
|
|
\arguments{
|
|
\item{obj}{\code{robxp} object}
|
|
|
|
\item{path}{path to test}
|
|
|
|
\item{user_agent}{user agent to test}
|
|
}
|
|
\description{
|
|
Test URL path against robots.txt
|
|
}
|
|
\examples{
|
|
library(robotstxt)
|
|
can_fetch(rt, "/asthma/asthma_stats/default.htm", "*") # TRUE
|
|
can_fetch(rt, "/_borders", "*") # FALSE
|
|
}
|
|
|