You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
29 lines
849 B
29 lines
849 B
% Generated by roxygen2: do not edit by hand
|
|
% Please edit documentation in R/crawl-delay.r
|
|
\name{crawl_delays}
|
|
\alias{crawl_delays}
|
|
\title{Retrive all agent crawl delay values in a \code{robxp} \code{robots.txt} object}
|
|
\usage{
|
|
crawl_delays(obj)
|
|
}
|
|
\arguments{
|
|
\item{obj}{\code{robxp} object}
|
|
}
|
|
\value{
|
|
data frame of agents and their crawl delays
|
|
}
|
|
\description{
|
|
Retrive all agent crawl delay values in a \code{robxp} \code{robots.txt} object
|
|
}
|
|
\note{
|
|
\code{-1} will be returned for any listed agent \emph{without} a crawl delay setting
|
|
}
|
|
\examples{
|
|
gh <- paste0(readLines(system.file("extdata", "github-robots.txt", package="spiderbar")), collapse="\\n")
|
|
gh_rt <- robxp(gh)
|
|
crawl_delays(gh_rt)
|
|
|
|
imdb <- paste0(readLines(system.file("extdata", "imdb-robots.txt", package="spiderbar")), collapse="\\n")
|
|
imdb_rt <- robxp(imdb)
|
|
crawl_delays(imdb_rt)
|
|
}
|
|
|