Browse Source
Merge pull request #4 from petermeissner/fix_partial_match
Partial parameter match in can_fetch() -- fixes #3
master
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with
3 additions and
3 deletions
-
DESCRIPTION
-
R/can-fetch.r
|
|
@ -1,8 +1,8 @@ |
|
|
|
Package: spiderbar |
|
|
|
Type: Package |
|
|
|
Title: Parse and Test Robots Exclusion Protocol Files and Rules |
|
|
|
Version: 0.2.2 |
|
|
|
Date: 2019-08-18 |
|
|
|
Version: 0.2.3 |
|
|
|
Date: 2020-05-29 |
|
|
|
Author: Bob Rudis (bob@rud.is) [aut, cre], SEOmoz, Inc [aut] |
|
|
|
Maintainer: Bob Rudis <bob@rud.is> |
|
|
|
Description: The 'Robots Exclusion Protocol' <https://www.robotstxt.org/orig.html> documents |
|
|
|
|
|
@ -22,7 +22,7 @@ |
|
|
|
can_fetch <- function(obj, path = "/", user_agent = "*") { |
|
|
|
|
|
|
|
if (inherits(obj, "robxp")) { |
|
|
|
vapply(path, rep_path_allowed, logical(1), x=obj, agent=user_agent, USE.NAMES=FALSE) |
|
|
|
vapply(path, rep_path_allowed, logical(1), xp=obj, agent=user_agent, USE.NAMES=FALSE) |
|
|
|
} else { |
|
|
|
return(NULL) |
|
|
|
} |
|
|
|