Browse Source

removed clipr

master
boB Rudis 2 years ago
parent
commit
36ce5660a6
No known key found for this signature in database GPG Key ID: 1D7529BE14E2BBA9
48 changed files with 61 additions and 64 deletions
  1. +6
    -6
      DESCRIPTION
  2. +0
    -1
      NAMESPACE
  3. +4
    -0
      NEWS.md
  4. +2
    -6
      R/as_req.r
  5. +0
    -1
      R/splashr-package.R
  6. +6
    -44
      cran-comments.md
  7. +2
    -5
      man/as_httr_req.Rd
  8. +1
    -0
      man/execute_lua.Rd
  9. +1
    -0
      man/get_content_size.Rd
  10. +1
    -0
      man/get_content_type.Rd
  11. +1
    -0
      man/get_har_entry.Rd
  12. +1
    -0
      man/get_request_type.Rd
  13. +1
    -0
      man/get_request_url.Rd
  14. +1
    -0
      man/get_response_body.Rd
  15. +1
    -0
      man/har_entry_count.Rd
  16. +1
    -0
      man/install_splash.Rd
  17. +1
    -0
      man/render_har.Rd
  18. +1
    -0
      man/render_html.Rd
  19. +1
    -0
      man/render_jpeg.Rd
  20. +1
    -0
      man/render_json.Rd
  21. +1
    -0
      man/render_png.Rd
  22. +1
    -0
      man/splash_active.Rd
  23. +1
    -0
      man/splash_add_lua.Rd
  24. +1
    -0
      man/splash_click.Rd
  25. +1
    -0
      man/splash_debug.Rd
  26. +1
    -0
      man/splash_enable_javascript.Rd
  27. +1
    -0
      man/splash_focus.Rd
  28. +1
    -0
      man/splash_go.Rd
  29. +1
    -0
      man/splash_har.Rd
  30. +1
    -0
      man/splash_har_reset.Rd
  31. +1
    -0
      man/splash_history.Rd
  32. +1
    -0
      man/splash_html.Rd
  33. +1
    -0
      man/splash_images.Rd
  34. +1
    -0
      man/splash_perf_stats.Rd
  35. +1
    -0
      man/splash_plugins.Rd
  36. +1
    -0
      man/splash_png.Rd
  37. +1
    -0
      man/splash_press.Rd
  38. +1
    -0
      man/splash_private_mode.Rd
  39. +1
    -0
      man/splash_release.Rd
  40. +1
    -0
      man/splash_response_body.Rd
  41. +1
    -0
      man/splash_send_keys.Rd
  42. +1
    -0
      man/splash_send_text.Rd
  43. +1
    -0
      man/splash_user_agent.Rd
  44. +1
    -0
      man/splash_version.Rd
  45. +1
    -0
      man/splash_wait.Rd
  46. +1
    -0
      man/start_splash.Rd
  47. +1
    -0
      man/stop_splash.Rd
  48. +1
    -1
      vignettes/intro_to_splashr.Rmd

+ 6
- 6
DESCRIPTION View File

@@ -1,12 +1,13 @@
Package: splashr
Type: Package
Title: Tools to Work with the 'Splash' 'JavaScript' Rendering and Scraping Service
Version: 0.4.0
Date: 2017-08-26
Version: 0.4.1
Date: 2018-01-16
Encoding: UTF-8
Authors@R: c(
person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"))
)
person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"),
comment = c(ORCID = "0000-0001-5670-2640"))
)
Maintainer: Bob Rudis <bob@rud.is>
Description: 'Splash' <https://github.com/scrapinghub/splash> is a 'JavaScript' rendering service.
It is a lightweight web browser with an 'HTTP' API, implemented in 'Python' using 'Twisted'
@@ -32,7 +33,6 @@ Imports:
xml2,
curl,
httr,
clipr,
purrr,
stats,
utils,
@@ -45,5 +45,5 @@ Imports:
jsonlite,
HARtools,
lubridate
RoxygenNote: 6.0.1
RoxygenNote: 6.0.1.9000
VignetteBuilder: knitr

+ 0
- 1
NAMESPACE View File

@@ -92,7 +92,6 @@ importFrom(HARtools,HARviewer)
importFrom(HARtools,HARviewerOutput)
importFrom(HARtools,renderHARviewer)
importFrom(HARtools,writeHAR)
importFrom(clipr,read_clip)
importFrom(curl,curl_unescape)
importFrom(formatR,tidy_source)
importFrom(jsonlite,fromJSON)


+ 4
- 0
NEWS.md View File

@@ -1,3 +1,7 @@
0.4.1

* removed clipr usage due to CRAN

0.4.0

* moved to 'docker' pacakge since it's on CRAN


+ 2
- 6
R/as_req.r View File

@@ -3,15 +3,13 @@
#' This function is very useful if you used `splashr` to find XHR requests in a dynamic
#' page and want to be able to make a call directly to that XHR resource. Once you
#' identify the proper HAR entry, pass it to this function and fully working function
#' that makes an `httr::VERB()` request will be created and returned. The text of the function
#' will also be put onto the clipboad if `add_clip` is `TRUE``
#' that makes an `httr::VERB()` request will be created and returned.
#'
#' @md
#' @param entry HAR entry
#' @param quiet quiet (no messages)
#' @param add_clip add clip (paste the function text to the clipboard)
#' @export
as_httr_req <- function(entry, quiet=TRUE, add_clip=TRUE) {
as_httr_req <- function(entry, quiet=TRUE) {

req <- entry$request

@@ -82,8 +80,6 @@ as_httr_req <- function(entry, quiet=TRUE, add_clip=TRUE) {
formatR::tidy_source(text=out, width.cutoff=30, indent=4, file=fil)
tmp <- paste0(readLines(fil), collapse="\n")

if (add_clip) clipr::write_clip(tmp)

if (!quiet) cat(tmp, "\n")

# make a bona fide R function


+ 0
- 1
R/splashr-package.R View File

@@ -20,7 +20,6 @@
#' @importFrom xml2 read_html url_parse
#' @importFrom jsonlite fromJSON unbox stream_in
#' @importFrom openssl base64_decode
#' @importFrom clipr read_clip
#' @importFrom lubridate ymd_hms
#' @importFrom scales comma
#' @importFrom stats setNames


+ 6
- 44
cran-comments.md View File

@@ -1,8 +1,8 @@
## Test environments

* local OS X install, R 3.4.1 on both 10.12 and 10.13 Beta 6
* local ubuntu 3.4.1 and r-devel
* ubuntu on travis-ci, R oldrel, 3.4.1 and r-devel
* local OS X install, R 3.4.3 on both 10.12 and 10.13.2
* local ubuntu 3.4.2 and r-devel
* ubuntu on travis-ci, R oldrel, current and r-devel
* win-builder (devel and release)

## R CMD check results
@@ -17,45 +17,7 @@ This is a new release, so there are no reverse dependencies.

---

Hey folks,
Submitting patch due to CRAN note.

The tests skip on CRAN and the examples are \dontrun{} because
they require an active Splash server running for the most
accurate results (mocking would not exercise the interaction with
the Splash API server). The core functionality is much the same
as RSelenium, but Splash is more designed for scraping
than Selenium which is more geared towards web site automated
testing. This will hopefully be a nice addition to the R web
scraping world.

It is fully tested on Travis: https://travis-ci.org/hrbrmstr/splashr
with 2 different Python configurations across all three R versions.
It's an active test that spins up a local Docker container and runs
on push and weekly. (It imports the 'docker' package, hence my
mentioning the diverse python configs).

Neither Appveyor (and, all other free Windows CI systems) nor
rhub nor WinBuilder support co-runbninbg Docker linux containers
so the tests are not run there, but the package is built regularly
on WinBuilder and rhub's Windows and "CRAN" environments to
catch any other platform-related package errors.

There are many Imports because this works with a diverse amount
of web content and also tries to play well with 'httr' and 'rvest'
as well as the 'hartools' package.

There are three vignettes and I tried to reduce the image sizes
for the figures in them as much as possible. I'll glady do some
cropping if you would like the total size to be even smaller. I
fully understand wanting to keep pacakges as small as possible.

The package is also well-documented and has 44% code coverage:
https://codecov.io/gh/hrbrmstr/splashr/tree/master/R
I'm going to get that up closer to 80% for the next release which will
extend the "DSL" functions.

Thx for your time & efforts!

-boB

P.S. Hope you folks had a great summer!
Removed clipboard functionality since that was the path of
least resistance.

+ 2
- 5
man/as_httr_req.Rd View File

@@ -4,19 +4,16 @@
\alias{as_httr_req}
\title{Create an httr verb request function from an HAR request}
\usage{
as_httr_req(entry, quiet = TRUE, add_clip = TRUE)
as_httr_req(entry, quiet = TRUE)
}
\arguments{
\item{entry}{HAR entry}

\item{quiet}{quiet (no messages)}

\item{add_clip}{add clip (paste the function text to the clipboard)}
}
\description{
This function is very useful if you used \code{splashr} to find XHR requests in a dynamic
page and want to be able to make a call directly to that XHR resource. Once you
identify the proper HAR entry, pass it to this function and fully working function
that makes an \code{httr::VERB()} request will be created and returned. The text of the function
will also be put onto the clipboad if \code{add_clip} is `TRUE``
that makes an \code{httr::VERB()} request will be created and returned.
}

+ 1
- 0
man/execute_lua.Rd View File

@@ -58,3 +58,4 @@ Other splash_renderers: \code{\link{render_har}},
\code{\link{render_html}}, \code{\link{render_jpeg}},
\code{\link{render_json}}, \code{\link{render_png}}
}
\concept{splash_renderers}

+ 1
- 0
man/get_content_size.Rd View File

@@ -26,3 +26,4 @@ Other splash_har_helpers: \code{\link{get_content_type}},
\code{\link{get_response_body}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

+ 1
- 0
man/get_content_type.Rd View File

@@ -61,3 +61,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_response_body}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

+ 1
- 0
man/get_har_entry.Rd View File

@@ -22,3 +22,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_response_body}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

+ 1
- 0
man/get_request_type.Rd View File

@@ -26,3 +26,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_response_body}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

+ 1
- 0
man/get_request_url.Rd View File

@@ -20,3 +20,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_response_body}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

+ 1
- 0
man/get_response_body.Rd View File

@@ -26,3 +26,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_request_url}},
\code{\link{har_entry_count}}
}
\concept{splash_har_helpers}

+ 1
- 0
man/har_entry_count.Rd View File

@@ -20,3 +20,4 @@ Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_request_url}},
\code{\link{get_response_body}}
}
\concept{splash_har_helpers}

+ 1
- 0
man/install_splash.Rd View File

@@ -23,3 +23,4 @@ stop_splash(splash_container)
Other splash_docker_helpers: \code{\link{start_splash}},
\code{\link{stop_splash}}
}
\concept{splash_docker_helpers}

+ 1
- 0
man/render_har.Rd View File

@@ -77,3 +77,4 @@ Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_html}}, \code{\link{render_jpeg}},
\code{\link{render_json}}, \code{\link{render_png}}
}
\concept{splash_renderers}

+ 1
- 0
man/render_html.Rd View File

@@ -78,3 +78,4 @@ Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_jpeg}},
\code{\link{render_json}}, \code{\link{render_png}}
}
\concept{splash_renderers}

+ 1
- 0
man/render_jpeg.Rd View File

@@ -82,3 +82,4 @@ Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_html}},
\code{\link{render_json}}, \code{\link{render_png}}
}
\concept{splash_renderers}

+ 1
- 0
man/render_json.Rd View File

@@ -114,3 +114,4 @@ Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_html}},
\code{\link{render_jpeg}}, \code{\link{render_png}}
}
\concept{splash_renderers}

+ 1
- 0
man/render_png.Rd View File

@@ -83,3 +83,4 @@ Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_html}},
\code{\link{render_jpeg}}, \code{\link{render_json}}
}
\concept{splash_renderers}

+ 1
- 0
man/splash_active.Rd View File

@@ -27,3 +27,4 @@ Other splash_info_functions: \code{\link{splash_debug}},
\code{\link{splash_perf_stats}},
\code{\link{splash_version}}
}
\concept{splash_info_functions}

+ 1
- 0
man/splash_add_lua.Rd View File

@@ -35,3 +35,4 @@ Other splash_dsl_functions: \code{\link{splash_click}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_click.Rd View File

@@ -23,3 +23,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_debug.Rd View File

@@ -27,4 +27,5 @@ Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_perf_stats}},
\code{\link{splash_version}}
}
\concept{splash_info_functions}
\keyword{internal}

+ 1
- 0
man/splash_enable_javascript.Rd View File

@@ -32,3 +32,4 @@ Other splash_dsl_attributes: \code{\link{splash_images}},
\code{\link{splash_private_mode}},
\code{\link{splash_response_body}}
}
\concept{splash_dsl_attributes}

+ 1
- 0
man/splash_focus.Rd View File

@@ -26,3 +26,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_go.Rd View File

@@ -34,3 +34,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_har.Rd View File

@@ -32,3 +32,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_har_reset.Rd View File

@@ -21,3 +21,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_history.Rd View File

@@ -24,3 +24,4 @@ Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_perf_stats}},
\code{\link{splash_version}}
}
\concept{splash_info_functions}

+ 1
- 0
man/splash_html.Rd View File

@@ -34,3 +34,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_images.Rd View File

@@ -33,3 +33,4 @@ Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_private_mode}},
\code{\link{splash_response_body}}
}
\concept{splash_dsl_attributes}

+ 1
- 0
man/splash_perf_stats.Rd View File

@@ -23,3 +23,4 @@ Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_debug}}, \code{\link{splash_history}},
\code{\link{splash_version}}
}
\concept{splash_info_functions}

+ 1
- 0
man/splash_plugins.Rd View File

@@ -30,3 +30,4 @@ Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_private_mode}},
\code{\link{splash_response_body}}
}
\concept{splash_dsl_attributes}

+ 1
- 0
man/splash_png.Rd View File

@@ -34,3 +34,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_press.Rd View File

@@ -23,3 +23,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_private_mode.Rd View File

@@ -33,3 +33,4 @@ Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_plugins}},
\code{\link{splash_response_body}}
}
\concept{splash_dsl_attributes}

+ 1
- 0
man/splash_release.Rd View File

@@ -23,3 +23,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_response_body.Rd View File

@@ -30,3 +30,4 @@ Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_plugins}},
\code{\link{splash_private_mode}}
}
\concept{splash_dsl_attributes}

+ 1
- 0
man/splash_send_keys.Rd View File

@@ -32,3 +32,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_release}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_send_text.Rd View File

@@ -29,3 +29,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_release}},
\code{\link{splash_send_keys}}, \code{\link{splash_wait}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/splash_user_agent.Rd View File

@@ -67,4 +67,5 @@ splash_local \%>\%
jsonlite::fromJSON()
}
}
\concept{splash_dsl_functions_functions}
\keyword{datasets}

+ 1
- 0
man/splash_version.Rd View File

@@ -23,3 +23,4 @@ Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_debug}}, \code{\link{splash_history}},
\code{\link{splash_perf_stats}}
}
\concept{splash_info_functions}

+ 1
- 0
man/splash_wait.Rd View File

@@ -34,3 +34,4 @@ Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}
}
\concept{splash_dsl_functions}

+ 1
- 0
man/start_splash.Rd View File

@@ -33,3 +33,4 @@ stop_splash(splash_container)
Other splash_docker_helpers: \code{\link{install_splash}},
\code{\link{stop_splash}}
}
\concept{splash_docker_helpers}

+ 1
- 0
man/stop_splash.Rd View File

@@ -28,3 +28,4 @@ stop_splash(splash_container)
Other splash_docker_helpers: \code{\link{install_splash}},
\code{\link{start_splash}}
}
\concept{splash_docker_helpers}

+ 1
- 1
vignettes/intro_to_splashr.Rmd View File

@@ -269,7 +269,7 @@ req() %>%
## Output is the same as previous block
```

The text of the function is also put on the clipboard by default, so you can paste it right into a script or package for use later on:
This is an example of the built `httr` function:

```
httr::VERB(verb = "GET", url = "https://analytics.usa.gov/data/live/devices.json",


Loading…
Cancel
Save