boB Rudis 6 роки тому
джерело
коміт
36ce5660a6
Не вдалося знайти GPG ключ що відповідає даному підпису Ідентифікатор GPG ключа: 1D7529BE14E2BBA9
  1. 12
      DESCRIPTION
  2. 1
      NAMESPACE
  3. 4
      NEWS.md
  4. 8
      R/as_req.r
  5. 1
      R/splashr-package.R
  6. 50
      cran-comments.md
  7. 7
      man/as_httr_req.Rd
  8. 1
      man/execute_lua.Rd
  9. 1
      man/get_content_size.Rd
  10. 1
      man/get_content_type.Rd
  11. 1
      man/get_har_entry.Rd
  12. 1
      man/get_request_type.Rd
  13. 1
      man/get_request_url.Rd
  14. 1
      man/get_response_body.Rd
  15. 1
      man/har_entry_count.Rd
  16. 1
      man/install_splash.Rd
  17. 1
      man/render_har.Rd
  18. 1
      man/render_html.Rd
  19. 1
      man/render_jpeg.Rd
  20. 1
      man/render_json.Rd
  21. 1
      man/render_png.Rd
  22. 1
      man/splash_active.Rd
  23. 1
      man/splash_add_lua.Rd
  24. 1
      man/splash_click.Rd
  25. 1
      man/splash_debug.Rd
  26. 1
      man/splash_enable_javascript.Rd
  27. 1
      man/splash_focus.Rd
  28. 1
      man/splash_go.Rd
  29. 1
      man/splash_har.Rd
  30. 1
      man/splash_har_reset.Rd
  31. 1
      man/splash_history.Rd
  32. 1
      man/splash_html.Rd
  33. 1
      man/splash_images.Rd
  34. 1
      man/splash_perf_stats.Rd
  35. 1
      man/splash_plugins.Rd
  36. 1
      man/splash_png.Rd
  37. 1
      man/splash_press.Rd
  38. 1
      man/splash_private_mode.Rd
  39. 1
      man/splash_release.Rd
  40. 1
      man/splash_response_body.Rd
  41. 1
      man/splash_send_keys.Rd
  42. 1
      man/splash_send_text.Rd
  43. 1
      man/splash_user_agent.Rd
  44. 1
      man/splash_version.Rd
  45. 1
      man/splash_wait.Rd
  46. 1
      man/start_splash.Rd
  47. 1
      man/stop_splash.Rd
  48. 2
      vignettes/intro_to_splashr.Rmd

12
DESCRIPTION

@ -1,12 +1,13 @@
Package: splashr
Type: Package
Title: Tools to Work with the 'Splash' 'JavaScript' Rendering and Scraping Service
Version: 0.4.0
Date: 2017-08-26
Version: 0.4.1
Date: 2018-01-16
Encoding: UTF-8
Authors@R: c(
person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"))
)
person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"),
comment = c(ORCID = "0000-0001-5670-2640"))
)
Maintainer: Bob Rudis <bob@rud.is>
Description: 'Splash' <https://github.com/scrapinghub/splash> is a 'JavaScript' rendering service.
It is a lightweight web browser with an 'HTTP' API, implemented in 'Python' using 'Twisted'
@ -32,7 +33,6 @@ Imports:
xml2,
curl,
httr,
clipr,
purrr,
stats,
utils,
@ -45,5 +45,5 @@ Imports:
jsonlite,
HARtools,
lubridate
RoxygenNote: 6.0.1
RoxygenNote: 6.0.1.9000
VignetteBuilder: knitr

1
NAMESPACE

@ -92,7 +92,6 @@ importFrom(HARtools,HARviewer)
importFrom(HARtools,HARviewerOutput)
importFrom(HARtools,renderHARviewer)
importFrom(HARtools,writeHAR)
importFrom(clipr,read_clip)
importFrom(curl,curl_unescape)
importFrom(formatR,tidy_source)
importFrom(jsonlite,fromJSON)

4
NEWS.md

@ -1,3 +1,7 @@
0.4.1
* removed clipr usage due to CRAN
0.4.0
* moved to 'docker' pacakge since it's on CRAN

8
R/as_req.r

@ -3,15 +3,13 @@
#' This function is very useful if you used `splashr` to find XHR requests in a dynamic
#' page and want to be able to make a call directly to that XHR resource. Once you
#' identify the proper HAR entry, pass it to this function and fully working function
#' that makes an `httr::VERB()` request will be created and returned. The text of the function
#' will also be put onto the clipboad if `add_clip` is `TRUE``
#' that makes an `httr::VERB()` request will be created and returned.
#'
#' @md
#' @param entry HAR entry
#' @param quiet quiet (no messages)
#' @param add_clip add clip (paste the function text to the clipboard)
#' @export
as_httr_req <- function(entry, quiet=TRUE, add_clip=TRUE) {
as_httr_req <- function(entry, quiet=TRUE) {
req <- entry$request
@ -82,8 +80,6 @@ as_httr_req <- function(entry, quiet=TRUE, add_clip=TRUE) {
formatR::tidy_source(text=out, width.cutoff=30, indent=4, file=fil)
tmp <- paste0(readLines(fil), collapse="\n")
if (add_clip) clipr::write_clip(tmp)
if (!quiet) cat(tmp, "\n")
# make a bona fide R function

1
R/splashr-package.R

@ -20,7 +20,6 @@
#' @importFrom xml2 read_html url_parse
#' @importFrom jsonlite fromJSON unbox stream_in
#' @importFrom openssl base64_decode
#' @importFrom clipr read_clip
#' @importFrom lubridate ymd_hms
#' @importFrom scales comma
#' @importFrom stats setNames

50
cran-comments.md

@ -1,8 +1,8 @@
## Test environments
* local OS X install, R 3.4.1 on both 10.12 and 10.13 Beta 6
* local ubuntu 3.4.1 and r-devel
* ubuntu on travis-ci, R oldrel, 3.4.1 and r-devel
* local OS X install, R 3.4.3 on both 10.12 and 10.13.2
* local ubuntu 3.4.2 and r-devel
* ubuntu on travis-ci, R oldrel, current and r-devel
* win-builder (devel and release)
## R CMD check results
@ -17,45 +17,7 @@ This is a new release, so there are no reverse dependencies.
---
Hey folks,
Submitting patch due to CRAN note.
The tests skip on CRAN and the examples are \dontrun{} because
they require an active Splash server running for the most
accurate results (mocking would not exercise the interaction with
the Splash API server). The core functionality is much the same
as RSelenium, but Splash is more designed for scraping
than Selenium which is more geared towards web site automated
testing. This will hopefully be a nice addition to the R web
scraping world.
It is fully tested on Travis: https://travis-ci.org/hrbrmstr/splashr
with 2 different Python configurations across all three R versions.
It's an active test that spins up a local Docker container and runs
on push and weekly. (It imports the 'docker' package, hence my
mentioning the diverse python configs).
Neither Appveyor (and, all other free Windows CI systems) nor
rhub nor WinBuilder support co-runbninbg Docker linux containers
so the tests are not run there, but the package is built regularly
on WinBuilder and rhub's Windows and "CRAN" environments to
catch any other platform-related package errors.
There are many Imports because this works with a diverse amount
of web content and also tries to play well with 'httr' and 'rvest'
as well as the 'hartools' package.
There are three vignettes and I tried to reduce the image sizes
for the figures in them as much as possible. I'll glady do some
cropping if you would like the total size to be even smaller. I
fully understand wanting to keep pacakges as small as possible.
The package is also well-documented and has 44% code coverage:
https://codecov.io/gh/hrbrmstr/splashr/tree/master/R
I'm going to get that up closer to 80% for the next release which will
extend the "DSL" functions.
Thx for your time & efforts!
-boB
P.S. Hope you folks had a great summer!
Removed clipboard functionality since that was the path of
least resistance.

7
man/as_httr_req.Rd

@ -4,19 +4,16 @@
\alias{as_httr_req}
\title{Create an httr verb request function from an HAR request}
\usage{
as_httr_req(entry, quiet = TRUE, add_clip = TRUE)
as_httr_req(entry, quiet = TRUE)
}
\arguments{
\item{entry}{HAR entry}
\item{quiet}{quiet (no messages)}
\item{add_clip}{add clip (paste the function text to the clipboard)}
}
\description{
This function is very useful if you used \code{splashr} to find XHR requests in a dynamic
page and want to be able to make a call directly to that XHR resource. Once you
identify the proper HAR entry, pass it to this function and fully working function
that makes an \code{httr::VERB()} request will be created and returned. The text of the function
will also be put onto the clipboad if \code{add_clip} is `TRUE``
that makes an \code{httr::VERB()} request will be created and returned.
}

1
man/execute_lua.Rd

@ -58,3 +58,4 @@ Other splash_renderers: \code{\link{render_har}},
\code{\link{render_html}}, \code{\link{render_jpeg}},
\code{\link{render_json}}, \code{\link{render_png}}
}
\concept{splash_renderers}

1
man/get_content_size.Rd

@ -26,3 +26,4 @@ Other splash_har_helpers: \code{\link{get_content_type}},
\code{\link{get_response_body}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

1
man/get_content_type.Rd

@ -61,3 +61,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_response_body}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

1
man/get_har_entry.Rd

@ -22,3 +22,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_response_body}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

1
man/get_request_type.Rd

@ -26,3 +26,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_response_body}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

1
man/get_request_url.Rd

@ -20,3 +20,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_response_body}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

1
man/get_response_body.Rd

@ -26,3 +26,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_request_url}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

1
man/har_entry_count.Rd

@ -20,3 +20,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_request_url}},
\code{\link{get_response_body}}
}
\concept{splash_har_helpers}

1
man/install_splash.Rd

@ -23,3 +23,4 @@ stop_splash(splash_container)
Other splash_docker_helpers: \code{\link{start_splash}},
\code{\link{stop_splash}}
}
\concept{splash_docker_helpers}

1
man/render_har.Rd

@ -77,3 +77,4 @@ Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_html}}, \code{\link{render_jpeg}},
\code{\link{render_json}}, \code{\link{render_png}}
}
\concept{splash_renderers}

1
man/render_html.Rd

@ -78,3 +78,4 @@ Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_jpeg}},
\code{\link{render_json}}, \code{\link{render_png}}
}
\concept{splash_renderers}

1
man/render_jpeg.Rd

@ -82,3 +82,4 @@ Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_html}},
\code{\link{render_json}}, \code{\link{render_png}}
}
\concept{splash_renderers}

1
man/render_json.Rd

@ -114,3 +114,4 @@ Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_html}},
\code{\link{render_jpeg}}, \code{\link{render_png}}
}
\concept{splash_renderers}

1
man/render_png.Rd

@ -83,3 +83,4 @@ Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_html}},
\code{\link{render_jpeg}}, \code{\link{render_json}}
}
\concept{splash_renderers}

1
man/splash_active.Rd

@ -27,3 +27,4 @@ Other splash_info_functions: \code{\link{splash_debug}},
\code{\link{splash_perf_stats}},
\code{\link{splash_version}}
}
\concept{splash_info_functions}

1
man/splash_add_lua.Rd

@ -35,3 +35,4 @@ Other splash_dsl_functions: \code{\link{splash_click}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_click.Rd

@ -23,3 +23,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_debug.Rd

@ -27,4 +27,5 @@ Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_perf_stats}},
\code{\link{splash_version}}
}
\concept{splash_info_functions}
\keyword{internal}

1
man/splash_enable_javascript.Rd

@ -32,3 +32,4 @@ Other splash_dsl_attributes: \code{\link{splash_images}},
\code{\link{splash_private_mode}},
\code{\link{splash_response_body}}
}
\concept{splash_dsl_attributes}

1
man/splash_focus.Rd

@ -26,3 +26,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_go.Rd

@ -34,3 +34,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_har.Rd

@ -32,3 +32,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_har_reset.Rd

@ -21,3 +21,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_history.Rd

@ -24,3 +24,4 @@ Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_perf_stats}},
\code{\link{splash_version}}
}
\concept{splash_info_functions}

1
man/splash_html.Rd

@ -34,3 +34,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_images.Rd

@ -33,3 +33,4 @@ Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_private_mode}},
\code{\link{splash_response_body}}
}
\concept{splash_dsl_attributes}

1
man/splash_perf_stats.Rd

@ -23,3 +23,4 @@ Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_debug}}, \code{\link{splash_history}},
\code{\link{splash_version}}
}
\concept{splash_info_functions}

1
man/splash_plugins.Rd

@ -30,3 +30,4 @@ Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_private_mode}},
\code{\link{splash_response_body}}
}
\concept{splash_dsl_attributes}

1
man/splash_png.Rd

@ -34,3 +34,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_press.Rd

@ -23,3 +23,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_private_mode.Rd

@ -33,3 +33,4 @@ Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_plugins}},
\code{\link{splash_response_body}}
}
\concept{splash_dsl_attributes}

1
man/splash_release.Rd

@ -23,3 +23,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_response_body.Rd

@ -30,3 +30,4 @@ Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_plugins}},
\code{\link{splash_private_mode}}
}
\concept{splash_dsl_attributes}

1
man/splash_send_keys.Rd

@ -32,3 +32,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_release}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_send_text.Rd

@ -29,3 +29,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_release}},
\code{\link{splash_send_keys}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

1
man/splash_user_agent.Rd

@ -67,4 +67,5 @@ splash_local \%>\%
jsonlite::fromJSON()
}
}
\concept{splash_dsl_functions_functions}
\keyword{datasets}

1
man/splash_version.Rd

@ -23,3 +23,4 @@ Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_debug}}, \code{\link{splash_history}},
\code{\link{splash_perf_stats}}
}
\concept{splash_info_functions}

1
man/splash_wait.Rd

@ -34,3 +34,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}
}
\concept{splash_dsl_functions}

1
man/start_splash.Rd

@ -33,3 +33,4 @@ stop_splash(splash_container)
Other splash_docker_helpers: \code{\link{install_splash}},
\code{\link{stop_splash}}
}
\concept{splash_docker_helpers}

1
man/stop_splash.Rd

@ -28,3 +28,4 @@ stop_splash(splash_container)
Other splash_docker_helpers: \code{\link{install_splash}},
\code{\link{start_splash}}
}
\concept{splash_docker_helpers}

2
vignettes/intro_to_splashr.Rmd

@ -269,7 +269,7 @@ req() %>%
## Output is the same as previous block
```
The text of the function is also put on the clipboard by default, so you can paste it right into a script or package for use later on:
This is an example of the built `httr` function:
```
httr::VERB(verb = "GET", url = "https://analytics.usa.gov/data/live/devices.json",

Завантаження…
Відмінити
Зберегти