Browse Source

better info in README; tweaks to image renderers; splash_local default parameter to many render functions

master
boB Rudis 7 years ago
parent
commit
2ccd4da8cc
  1. 53
      R/docker.r
  2. 2
      R/render-har.r
  3. 2
      R/render-html.r
  4. 6
      R/render-jpg.r
  5. 2
      R/render-json.r
  6. 20
      R/render-png.r
  7. 2
      R/render_file.R
  8. 10
      R/splashr.r
  9. 53
      README.Rmd
  10. 138
      README.md
  11. BIN
      img/cap.jpg
  12. BIN
      img/cap.png
  13. 16
      man/install_splash.Rd
  14. 4
      man/render_file.Rd
  15. 9
      man/render_har.Rd
  16. 4
      man/render_html.Rd
  17. 4
      man/render_jpeg.Rd
  18. 4
      man/render_json.Rd
  19. 14
      man/render_png.Rd
  20. 2
      man/splash_active.Rd
  21. 2
      man/splash_debug.Rd
  22. 2
      man/splash_history.Rd
  23. 2
      man/splash_perf_stats.Rd
  24. 2
      man/splash_version.Rd
  25. 22
      man/start_splash.Rd
  26. 20
      man/stop_splash.Rd

53
R/docker.r

@ -1,53 +0,0 @@
#' Retrieve the Docker image for Splash
#'
#' @md
#' @param host Docker host; defauolts to `localhost`
#' @return `harbor` `host` object
#' @export
#' @examples \dontrun{
#' install_splash()
#' splash_container <- start_splash()
#' stop_splash(splash_container)
#' }
install_splash <- function(host = harbor::localhost) {
harbor::docker_pull(host, "scrapinghub/splash")
}
#' Start a Splash server Docker container
#'
#' @param host Docker host; defauolts to `localhost`
#' @note you need Docker running on your system and have pulled the container with
#' [install_spash] for this to work. You should save the resultant `host`
#' object for use in [stop_splash].
#' @return `harbor` `container` object
#' @export
#' @examples \dontrun{
#' install_splash()
#' splash_container <- start_splash()
#' stop_splash(splash_container)
#' }
start_splash <- function(host = harbor::localhost) {
harbor::docker_run(host,
image = "scrapinghub/splash",
detach = TRUE,
docker_opts = c("-p", "5023:5023",
"-p", "8050:8050",
"-p", "8051:8051"))
}
#' Stop a running a Splash server Docker container
#'
#' @param splash_container saved Splash container id from [start_splash]
#' @param splash_container Docker `container` object created by [start_splash]
#' @note you need Docker running on your system and have pulled the container with
#' [install_spash] and started the Splash container with [start_splash] for this
#' to work. You will need the `container` object from [start_splash] for this to work.
#' @export
#' @examples \dontrun{
#' install_splash()
#' splash_container <- start_splash()
#' stop_splash(splash_container)
#' }
stop_splash <- function(splash_container) {
harbor::container_rm(splash_container, force=TRUE)
}

2
R/render-har.r

@ -9,7 +9,7 @@
#' @return a [HARtools::har] object #' @return a [HARtools::har] object
#' @references [Splash docs](http://splash.readthedocs.io/en/stable/index.html) #' @references [Splash docs](http://splash.readthedocs.io/en/stable/index.html)
#' @export #' @export
render_har <- function(splash_obj, url, base_url, response_body=FALSE, timeout=30, resource_timeout, wait=0, render_har <- function(splash_obj = splash_local, url, base_url, response_body=FALSE, timeout=30, resource_timeout, wait=0,
proxy, js, js_src, filters, allowed_domains, allowed_content_types, proxy, js, js_src, filters, allowed_domains, allowed_content_types,
forbidden_content_types, viewport="1024x768", images, headers, body, forbidden_content_types, viewport="1024x768", images, headers, body,
http_method, save_args, load_args) { http_method, save_args, load_args) {

2
R/render-html.r

@ -30,7 +30,7 @@
#' character vector. #' character vector.
#' @references [Splash docs](http://splash.readthedocs.io/en/stable/index.html) #' @references [Splash docs](http://splash.readthedocs.io/en/stable/index.html)
#' @export #' @export
render_html <- function(splash_obj, url, base_url, timeout=30, resource_timeout, wait=0, render_html <- function(splash_obj = splash_local, url, base_url, timeout=30, resource_timeout, wait=0,
proxy, js, js_src, filters, allowed_domains, allowed_content_types, proxy, js, js_src, filters, allowed_domains, allowed_content_types,
forbidden_content_types, viewport="1024x768", images, headers, body, forbidden_content_types, viewport="1024x768", images, headers, body,
http_method, save_args, load_args, raw_html=FALSE) { http_method, save_args, load_args, raw_html=FALSE) {

6
R/render-jpg.r

@ -8,13 +8,15 @@
#' @references [Splash docs](http://splash.readthedocs.io/en/stable/index.html) #' @references [Splash docs](http://splash.readthedocs.io/en/stable/index.html)
#' @export #' @export
render_jpeg <- render_jpg <- function( render_jpeg <- render_jpg <- function(
splash_obj, url, base_url=NULL, quality=75, width=1024, height=768, splash_obj = splash_local, url, base_url=NULL, quality=75, width=1024, height=768,
timeout=30, resource_timeout, wait=0, render_all=FALSE, timeout=30, resource_timeout, wait=0, render_all=FALSE,
proxy, js, js_src, filters, allowed_domains, allowed_content_types, proxy, js, js_src, filters, allowed_domains, allowed_content_types,
forbidden_content_types, viewport="1024x768", images, headers, body, forbidden_content_types, viewport="1024x768", images, headers, body,
http_method, save_args, load_args) { http_method, save_args, load_args) {
params <- list(url=url, timeout=timeout, wait=wait, viewport=viewport, params <- list(url=url, timeout=timeout,
wait=if (render_all & wait == 0) 0.5 else wait,
viewport=viewport,
quality=quality, width=width, height=height, render_all=as.numeric(render_all)) quality=quality, width=width, height=height, render_all=as.numeric(render_all))
if (!missing(base_url)) params$base_url <- base_url if (!missing(base_url)) params$base_url <- base_url

2
R/render-json.r

@ -27,7 +27,7 @@
#' overwhelmed with data. Use [str] to inspect various portions of the result. #' overwhelmed with data. Use [str] to inspect various portions of the result.
#' @references [Splash docs](http://splash.readthedocs.io/en/stable/index.html) #' @references [Splash docs](http://splash.readthedocs.io/en/stable/index.html)
#' @export #' @export
render_json <- function(splash_obj, url, base_url=NULL, quality=75, width=1024, height=768, render_json <- function(splash_obj = splash_local, url, base_url=NULL, quality=75, width=1024, height=768,
timeout=30, resource_timeout, wait=0, render_all=FALSE, timeout=30, resource_timeout, wait=0, render_all=FALSE,
proxy, js, js_src, filters, allowed_domains, allowed_content_types, proxy, js, js_src, filters, allowed_domains, allowed_content_types,
forbidden_content_types, viewport="1024x768", images, headers, body, forbidden_content_types, viewport="1024x768", images, headers, body,

20
R/render-png.r

@ -7,14 +7,17 @@
#' @references [Splash docs](http://splash.readthedocs.io/en/stable/index.html) #' @references [Splash docs](http://splash.readthedocs.io/en/stable/index.html)
#' @inheritParams render_html #' @inheritParams render_html
#' @export #' @export
render_png <- function(splash_obj, url, base_url, width=1024, height=768, render_all=TRUE, render_png <- function(
timeout=30, resource_timeout, wait=0, splash_obj = splash_local, url, base_url=NULL, quality=75, width=1024, height=768,
proxy, js, js_src, filters, allowed_domains, allowed_content_types, timeout=30, resource_timeout, wait=0, render_all=FALSE,
forbidden_content_types, viewport="1024x768", images, headers, body, proxy, js, js_src, filters, allowed_domains, allowed_content_types,
http_method, save_args, load_args) { forbidden_content_types, viewport="1024x768", images, headers, body,
http_method, save_args, load_args) {
params <- list(url=url, timeout=timeout, wait=wait, viewport=viewport, params <- list(url=url, timeout=timeout,
width=width, height=height, render_all=as.numeric(render_all)) wait=if (render_all & wait == 0) 0.5 else wait,
viewport=viewport,
quality=quality, width=width, height=height, render_all=as.numeric(render_all))
if (!missing(base_url)) params$base_url <- base_url if (!missing(base_url)) params$base_url <- base_url
if (!missing(resource_timeout)) params$resource_timeout <- resource_timeout if (!missing(resource_timeout)) params$resource_timeout <- resource_timeout
@ -38,4 +41,5 @@ render_png <- function(splash_obj, url, base_url, width=1024, height=768, render
magick::image_read(httr::content(res, as="raw")) magick::image_read(httr::content(res, as="raw"))
} }

2
R/render_file.R

@ -13,7 +13,7 @@
#' @param ... other params to [render_html]() or [render_png]() #' @param ... other params to [render_html]() or [render_png]()
#' @return An XML document or `magick` object #' @return An XML document or `magick` object
#' @export #' @export
render_file <- function(splash_obj, file_path, output=c("html", "png"), wait=0, viewport="1024x768", ...) { render_file <- function(splash_obj = slpash_local, file_path, output=c("html", "png"), wait=0, viewport="1024x768", ...) {
output <- match.arg(output, c("html", "png")) output <- match.arg(output, c("html", "png"))

10
R/splashr.r

@ -21,7 +21,7 @@ s_GET <- purrr::safely(GET)
#' @param splash_obj A splash connection object #' @param splash_obj A splash connection object
#' @return `TRUE` if Slash server is running, otherwise `FALSE` #' @return `TRUE` if Slash server is running, otherwise `FALSE`
#' @export #' @export
splash_active <- function(splash_obj) { splash_active <- function(splash_obj = splash_local) {
res <- s_GET(splash_url(splash_obj), path="_ping") res <- s_GET(splash_url(splash_obj), path="_ping")
@ -46,7 +46,7 @@ splash_active <- function(splash_obj) {
#' #'
#' @param splash_obj A splash connection object #' @param splash_obj A splash connection object
#' @export #' @export
splash_version <- function(splash_obj) { splash_version <- function(splash_obj = splash_local) {
execute_lua(splash_obj, ' execute_lua(splash_obj, '
function main(splash) function main(splash)
return splash:get_version() return splash:get_version()
@ -59,7 +59,7 @@ end
#' #'
#' @param splash_obj A splash connection object #' @param splash_obj A splash connection object
#' @export #' @export
splash_history <- function(splash_obj) { splash_history <- function(splash_obj = splash_local) {
execute_lua(splash_obj, ' execute_lua(splash_obj, '
function main(splash) function main(splash)
return splash:history() return splash:history()
@ -73,7 +73,7 @@ end
#' #'
#' @param splash_obj A splash connection object #' @param splash_obj A splash connection object
#' @export #' @export
splash_perf_stats <- function(splash_obj) { splash_perf_stats <- function(splash_obj = splash_local) {
execute_lua(splash_obj, ' execute_lua(splash_obj, '
function main(splash) function main(splash)
return splash:get_perf_stats() return splash:get_perf_stats()
@ -86,7 +86,7 @@ end
#' #'
#' @param splash_obj A splash connection object #' @param splash_obj A splash connection object
#' @export #' @export
splash_debug <- function(splash_obj) { splash_debug <- function(splash_obj = splash_local) {
httr::GET(splash_url(splash_obj), path="_debug") %>% httr::GET(splash_url(splash_obj), path="_debug") %>%
httr::stop_for_status() %>% httr::stop_for_status() %>%

53
README.Rmd

@ -15,7 +15,14 @@ You can also get it running with two commands:
sudo docker pull hrbrmstr/splashttpd sudo docker pull hrbrmstr/splashttpd
sudo docker run -p 5023:5023 -p 8050:8050 -p 8051:8051 hrbrmstr/splashttpd sudo docker run -p 5023:5023 -p 8050:8050 -p 8051:8051 hrbrmstr/splashttpd
(Do whatever you Windows ppl do with Docker on your systems to make ^^ work.) Do whatever you Windows ppl do with Docker on your systems to make ^^ work.
Folks super-new to Docker on Unix-ish platforms should [make sure to do](https://github.com/hrbrmstr/splashr/issues/3#issuecomment-280686494):
sudo groupadd docker
sudo usermod -aG docker $USER
(`$USER` is your username and shld be defined for you in the environment)
If using the [`harbor`](https://github.com/wch/harbor) package you can use the convience wrappers in this pacakge: If using the [`harbor`](https://github.com/wch/harbor) package you can use the convience wrappers in this pacakge:
@ -26,7 +33,7 @@ and then run:
stop_splash(splash_container) stop_splash(splash_container)
when done. All of that happens on your localhost so use `localhost` as the Splash server parameter. when done. All of that happens on your localhost and you will not need to specify `splash_obj` to many of the `splashr` functions if you're running Splash in this default configuration as long as you use named parameters. You can also use the pre-defined `splash_local` object if you want to use positional parameters.
You can run Selenium in Docker, so this is not unique to Splash. But, a Docker context makes it so that you don't have to run or maintain icky Python stuff directly on your system. Leave it in the abandoned warehouse district where it belongs. You can run Selenium in Docker, so this is not unique to Splash. But, a Docker context makes it so that you don't have to run or maintain icky Python stuff directly on your system. Leave it in the abandoned warehouse district where it belongs.
@ -111,6 +118,8 @@ options(width=120)
### Usage ### Usage
NOTE: ALL of these examples assume Splash is running in the default configuraiton on `localhost` (i.e. started with `start_splash()` or the docker example commands) unless otherwise noted.
```{r message=FALSE, warning=FALSE, error=FALSE} ```{r message=FALSE, warning=FALSE, error=FALSE}
library(splashr) library(splashr)
library(magick) library(magick)
@ -124,27 +133,23 @@ library(tidyverse)
# current verison # current verison
packageVersion("splashr") packageVersion("splashr")
splash("splash", 8050L) %>% splash_active()
splash_active()
splash("splash", 8050L) %>% splash_debug()
splash_debug()
``` ```
Notice the difference between a rendered HTML scrape and a non-rendered one: Notice the difference between a rendered HTML scrape and a non-rendered one:
```{r} ```{r}
splash("splash", 8050L) %>% render_html(url = "http://marvel.com/universe/Captain_America_(Steve_Rogers)")
render_html("http://marvel.com/universe/Captain_America_(Steve_Rogers)")
read_html("http://marvel.com/universe/Captain_America_(Steve_Rogers)") xml2::read_html("http://marvel.com/universe/Captain_America_(Steve_Rogers)")
``` ```
You can also profile pages: You can also profile pages:
```{r} ```{r}
splash("splash", 8050L) %>% render_har(url = "http://www.poynter.org/") -> har
render_har("http://www.poynter.org/") -> har
print(har) print(har)
``` ```
@ -154,26 +159,22 @@ You can use [`HARtools::HARviewer`](https://github.com/johndharrison/HARtools/bl
Full web page snapshots are easy-peasy too: Full web page snapshots are easy-peasy too:
```{r eval=FALSE} ```{r eval=FALSE}
splash("splash", 8050L) %>% render_png(url = "http://www.marveldirectory.com/individuals/c/captainamerica.htm")
render_png("http://marvel.com/universe/Captain_America_(Steve_Rogers)")
``` ```
```{r eval=TRUE, include=FALSE} ```{r eval=TRUE, include=FALSE}
splash("splash", 8050L) %>% render_png(url = "http://www.marveldirectory.com/individuals/c/captainamerica.htm") %>%
render_png("http://marvel.com/universe/Captain_America_(Steve_Rogers)") %>%
image_write("img/cap.png") image_write("img/cap.png")
``` ```
![](img/cap.png) ![](img/cap.png)
```{r eval=FALSE} ```{r eval=FALSE}
splash("splash", 8050L) %>% render_jpeg(url = "http://static2.comicvine.com/uploads/scale_small/3/31666/5052983-capasr2016001-eptingvar-18bdb.jpg")
render_jpeg("http://marvel.com/universe/Captain_America_(Steve_Rogers)")
``` ```
```{r eval=TRUE, include=FALSE} ```{r eval=TRUE, include=FALSE}
splash("splash", 8050L) %>% render_jpeg(url = "http://static2.comicvine.com/uploads/scale_small/3/31666/5052983-capasr2016001-eptingvar-18bdb.jpg") %>%
render_jpeg("http://marvel.com/universe/Captain_America_(Steve_Rogers)") %>%
image_write("img/cap.jpg") image_write("img/cap.jpg")
``` ```
@ -191,7 +192,7 @@ function main(splash)
end end
' '
res <- splash("localhost") %>% execute_lua(lua_ex) splash_local %>% execute_lua(lua_ex) -> res
rawToChar(res) %>% rawToChar(res) %>%
jsonlite::fromJSON() jsonlite::fromJSON()
@ -200,10 +201,10 @@ rawToChar(res) %>%
### Rendering Widgets ### Rendering Widgets
```{r eval=FALSE} ```{r eval=FALSE}
splash_vm <- start_splash(add_tempdir=TRUE) splash_vm <- start_splash(add_tempdir = TRUE)
``` ```
```{r} ```{r eval=FALSE}
DiagrammeR(" DiagrammeR("
graph LR graph LR
A-->B A-->B
@ -216,13 +217,7 @@ DiagrammeR("
") %>% ") %>%
saveWidget("/tmp/diag.html") saveWidget("/tmp/diag.html")
splash("localhost") %>% render_file(url = "/tmp/diag.html", output="html")
render_file("/tmp/diag.html", output="html")
```
```{r eval=FALSE}
splash("localhost") %>%
render_file("/tmp/diag.html", output="png", wait=2)
``` ```
![](img/diag.png) ![](img/diag.png)

138
README.md

@ -12,7 +12,14 @@ You can also get it running with two commands:
sudo docker pull hrbrmstr/splashttpd sudo docker pull hrbrmstr/splashttpd
sudo docker run -p 5023:5023 -p 8050:8050 -p 8051:8051 hrbrmstr/splashttpd sudo docker run -p 5023:5023 -p 8050:8050 -p 8051:8051 hrbrmstr/splashttpd
(Do whatever you Windows ppl do with Docker on your systems to make ^^ work.) Do whatever you Windows ppl do with Docker on your systems to make ^^ work.
Folks super-new to Docker on Unix-ish platforms should [make sure to do](https://github.com/hrbrmstr/splashr/issues/3#issuecomment-280686494):
sudo groupadd docker
sudo usermod -aG docker $USER
(`$USER` is your username and shld be defined for you in the environment)
If using the [`harbor`](https://github.com/wch/harbor) package you can use the convience wrappers in this pacakge: If using the [`harbor`](https://github.com/wch/harbor) package you can use the convience wrappers in this pacakge:
@ -23,7 +30,7 @@ and then run:
stop_splash(splash_container) stop_splash(splash_container)
when done. All of that happens on your localhost so use `localhost` as the Splash server parameter. when done. All of that happens on your localhost and you will not need to specify `splash_obj` to many of the `splashr` functions if you're running Splash in this default configuration as long as you use named parameters. You can also use the pre-defined `splash_local` object if you want to use positional parameters.
You can run Selenium in Docker, so this is not unique to Splash. But, a Docker context makes it so that you don't have to run or maintain icky Python stuff directly on your system. Leave it in the abandoned warehouse district where it belongs. You can run Selenium in Docker, so this is not unique to Splash. But, a Docker context makes it so that you don't have to run or maintain icky Python stuff directly on your system. Leave it in the abandoned warehouse district where it belongs.
@ -31,7 +38,7 @@ All you need for this package to work is a running Splash instance. You provide
### About Splash ### About Splash
> 'Splash' <https://github.com/scrapinghub/splash> is a javascript rendering service. It’s a lightweight web browser with an 'HTTP' API, implemented in Python using 'Twisted'and 'QT' [and provides some of the core functionality of the 'RSelenium' or 'seleniumPipes' R packages but with a Java-free footprint]. The (twisted) 'QT' reactor is used to make the sever fully asynchronous allowing to take advantage of 'webkit' concurrency via QT main loop. Some of Splash features include the ability to process multiple webpages in parallel; retrieving HTML results and/or take screenshots; disabling images or use Adblock Plus rules to make rendering faster; executing custom JavaScript in page context; getting detailed rendering info in HAR format. > 'Splash' <https://github.com/scrapinghub/splash> is a javascript rendering service. It’s a lightweight web browser with an 'HTTP' API, implemented in Python using 'Twisted'and 'QT' \[and provides some of the core functionality of the 'RSelenium' or 'seleniumPipes' R packages but with a Java-free footprint\]. The (twisted) 'QT' reactor is used to make the sever fully asynchronous allowing to take advantage of 'webkit' concurrency via QT main loop. Some of Splash features include the ability to process multiple webpages in parallel; retrieving HTML results and/or take screenshots; disabling images or use Adblock Plus rules to make rendering faster; executing custom JavaScript in page context; getting detailed rendering info in HAR format.
The following functions are implemented: The following functions are implemented:
@ -48,40 +55,40 @@ The following functions are implemented:
Mini-DSL (domain-specific language). These can be used to create a "script" without actually scripting in Lua. They are a less-powerful/configurable set of calls than what you can make with a full Lua function but the idea is to have it take care of very common but simple use-cases, like waiting a period of time before capturing a HAR/HTML/PNG image of a site: Mini-DSL (domain-specific language). These can be used to create a "script" without actually scripting in Lua. They are a less-powerful/configurable set of calls than what you can make with a full Lua function but the idea is to have it take care of very common but simple use-cases, like waiting a period of time before capturing a HAR/HTML/PNG image of a site:
- `splash_plugins`: Enable or disable browser plugins (e.g. Flash). - `splash_plugins`: Enable or disable browser plugins (e.g. Flash).
- `splash_images`: Enable/disable images - `splash_images`: Enable/disable images
- `splash_response_body`: Enable or disable response content tracking. - `splash_response_body`: Enable or disable response content tracking.
- `splash_go`: Go to an URL. - `splash_go`: Go to an URL.
- `splash_wait`: Wait for a period time - `splash_wait`: Wait for a period time
- `splash_har`: Return information about Splash interaction with a website in HAR format. - `splash_har`: Return information about Splash interaction with a website in HAR format.
- `splash_html`: Return a HTML snapshot of a current page. - `splash_html`: Return a HTML snapshot of a current page.
- `splash_png`: Return a screenshot of a current page in PNG format. - `splash_png`: Return a screenshot of a current page in PNG format.
- `splash_user_agent: Overwrite the User-Agent header for all further requests. NOTE: There are many "helper" user agent strings to go with `splash_user_agent`. Look for objects in `splashr` starting with `ua_`. - `splash_user_agent: Overwrite the User-Agent header for all further requests. NOTE: There are many "helper" user agent strings to go with`splash\_user\_agent`. Look for objects in`splashr`starting with`ua\_\`.
`httr` helpers. These help turn various bits of `splashr` objects into `httr`-ish things: `httr` helpers. These help turn various bits of `splashr` objects into `httr`-ish things:
- `as_req`: Turn a HAR response entry into a working `httr` function you can use to make a request with - `as_req`: Turn a HAR response entry into a working `httr` function you can use to make a request with
- `as_request`: Turn a HAR response entry into an `httr` `response`-like object (i.e. you can use `httr::content()` on it) - `as_request`: Turn a HAR response entry into an `httr` `response`-like object (i.e. you can use `httr::content()` on it)
Helpers: Helpers:
- `get_body_size`: Retrieve size of content | body | headers - `get_body_size`: Retrieve size of content | body | headers
- `get_content_sie`: Retrieve size of content | body | headers - `get_content_sie`: Retrieve size of content | body | headers
- `get_content_type` Retrieve or test content type of a HAR request object - `get_content_type` Retrieve or test content type of a HAR request object
- `get_headers_size` Retrieve size of content | body | headers - `get_headers_size` Retrieve size of content | body | headers
- `is_binary`: Retrieve or test content type of a HAR request object - `is_binary`: Retrieve or test content type of a HAR request object
- `is_content_type`: Retrieve or test content type of a HAR request object - `is_content_type`: Retrieve or test content type of a HAR request object
- `is_css`: Retrieve or test content type of a HAR request object - `is_css`: Retrieve or test content type of a HAR request object
- `is_gif`: Retrieve or test content type of a HAR request object - `is_gif`: Retrieve or test content type of a HAR request object
- `is_html`: Retrieve or test content type of a HAR request object - `is_html`: Retrieve or test content type of a HAR request object
- `is_javascript`: Retrieve or test content type of a HAR request object - `is_javascript`: Retrieve or test content type of a HAR request object
- `is_jpeg`: Retrieve or test content type of a HAR request object - `is_jpeg`: Retrieve or test content type of a HAR request object
- `is_json`: Retrieve or test content type of a HAR request object - `is_json`: Retrieve or test content type of a HAR request object
- `is_plain`: Retrieve or test content type of a HAR request object - `is_plain`: Retrieve or test content type of a HAR request object
- `is_png`: Retrieve or test content type of a HAR request object - `is_png`: Retrieve or test content type of a HAR request object
- `is_svg`: Retrieve or test content type of a HAR request object - `is_svg`: Retrieve or test content type of a HAR request object
- `is_xhr`: Retrieve or test content type of a HAR request object - `is_xhr`: Retrieve or test content type of a HAR request object
- `is_xml`: Retrieve or test content type of a HAR request object - `is_xml`: Retrieve or test content type of a HAR request object
Some functions from `HARtools` are imported/exported and `%>%` is imported/exported. Some functions from `HARtools` are imported/exported and `%>%` is imported/exported.
@ -108,6 +115,8 @@ options(width=120)
### Usage ### Usage
NOTE: ALL of these examples assume Splash is running in the default configuraiton on `localhost` (i.e. started with `start_splash()` or the docker example commands) unless otherwise noted.
``` r ``` r
library(splashr) library(splashr)
library(magick) library(magick)
@ -125,46 +134,43 @@ packageVersion("splashr")
## [1] '0.3.0' ## [1] '0.3.0'
``` r ``` r
splash("splash", 8050L) %>% splash_active()
splash_active()
``` ```
## [1] TRUE ## [1] TRUE
``` r ``` r
splash("splash", 8050L) %>% splash_debug()
splash_debug()
``` ```
## List of 7 ## List of 7
## $ active : list() ## $ active : list()
## $ argcache: int 0 ## $ argcache: int 0
## $ fds : int 17 ## $ fds : int 21
## $ leaks :List of 4 ## $ leaks :List of 4
## ..$ Deferred : int 50 ## ..$ Deferred : int 50
## ..$ LuaRuntime: int 1 ## ..$ LuaRuntime: int 1
## ..$ QTimer : int 1 ## ..$ QTimer : int 1
## ..$ Request : int 1 ## ..$ Request : int 1
## $ maxrss : int 491092 ## $ maxrss : int 202020
## $ qsize : int 0 ## $ qsize : int 0
## $ url : chr "http://splash:8050" ## $ url : chr "http://localhost:8050"
## - attr(*, "class")= chr [1:2] "splash_debug" "list" ## - attr(*, "class")= chr [1:2] "splash_debug" "list"
## NULL ## NULL
Notice the difference between a rendered HTML scrape and a non-rendered one: Notice the difference between a rendered HTML scrape and a non-rendered one:
``` r ``` r
splash("splash", 8050L) %>% render_html(url = "http://marvel.com/universe/Captain_America_(Steve_Rogers)")
render_html("http://marvel.com/universe/Captain_America_(Steve_Rogers)")
``` ```
## {xml_document} ## {xml_document}
## <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> ## <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr">
## [1] <head>\n<script type="text/javascript" async="async" src="http://uncanny.marvel.com/id?callback=s_c_il%5B1%5D._se ... ## [1] <head>\n<script type="text/javascript" async="async" src="http://dpm.demdex.net/id?d_rtbd=json&amp;d_ver=2&amp;d_ ...
## [2] <body>\n<iframe src="http://tpc.googlesyndication.com/safeframe/1-0-5/html/container.html" style="visibility: hid ... ## [2] <body id="index-index" class="index-index" onload="findLinks('myLink');">\n\n\t<div id="page_frame" style="overfl ...
``` r ``` r
read_html("http://marvel.com/universe/Captain_America_(Steve_Rogers)") xml2::read_html("http://marvel.com/universe/Captain_America_(Steve_Rogers)")
``` ```
## {xml_document} ## {xml_document}
@ -175,8 +181,7 @@ read_html("http://marvel.com/universe/Captain_America_(Steve_Rogers)")
You can also profile pages: You can also profile pages:
``` r ``` r
splash("splash", 8050L) %>% render_har(url = "http://www.poynter.org/") -> har
render_har("http://www.poynter.org/") -> har
print(har) print(har)
``` ```
@ -192,36 +197,34 @@ print(har)
## --------HAR PAGES-------- ## --------HAR PAGES--------
## Page id: 1 , Page title: Poynter – A global leader in journalism. Strengthening democracy. ## Page id: 1 , Page title: Poynter – A global leader in journalism. Strengthening democracy.
## --------HAR ENTRIES-------- ## --------HAR ENTRIES--------
## Number of entries: 58 ## Number of entries: 18
## REQUESTS: ## REQUESTS:
## Page: 1 ## Page: 1
## Number of entries: 58 ## Number of entries: 18
## - http://www.poynter.org/ ## - http://www.poynter.org/
## - http://www.poynter.org/wp-content/plugins/easy-author-image/css/easy-author-image.css?ver=2016_06_24.1
## - http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css?ver=2016_06_24.1 ## - http://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css?ver=2016_06_24.1
## - http://cloud.webtype.com/css/162ac332-3b31-4b73-ad44-da375b7f2fe3.css?ver=2016_06_24.1 ## - http://www.poynter.org/wp-content/themes/poynter_timber/assets/scrollbar/jquery.mCustomScrollbar.min.css?ver=2016...
## - http://maxcdn.bootstrapcdn.com/font-awesome/4.4.0/css/font-awesome.min.css?ver=2016_06_24.1 ## - http://www.poynter.org/wp-content/plugins/jetpack/css/jetpack.css?ver=4.0.4
## - http://www.poynter.org/wp-content/themes/poynter_timber/style.css?ver=2016_06_24.1
## ........ ## ........
## - https://s1.2mdn.net/4633855/GSCS-Bnr-Static-FY17-JO-728x90.jpg ## - http://cloud.webtype.com/webtype/ff2/3/60a9547a-da5f-42cd-8e1f-769a99fc27cd?ec_token=8f7c4c4997246fd7fa920171c994...
## - https://tpc.googlesyndication.com/simgad/9335380309894607046 ## - http://maxcdn.bootstrapcdn.com/font-awesome/4.4.0/fonts/fontawesome-webfont.woff?v=4.4.0
## - https://securepubads.g.doubleclick.net/pcs/view?xai=AKAOjstJ-3akL2MRC2PYT_DulENf77-tUEfP0yypiv4OS4ub1_Ojuauj-DY9e... ## - http://cloud.webtype.com/webtype/ff2/3/7dac2f83-affe-4762-81f1-056f47b74dc7?ec_token=8f7c4c4997246fd7fa920171c994...
## - https://tpc.googlesyndication.com/simgad/14632508855395020440 ## - http://cloud.webtype.com/webtype/ff2/3/4ac7f809-9bdf-4acc-8bd5-a922f05f271b?ec_token=8f7c4c4997246fd7fa920171c994...
## - https://securepubads.g.doubleclick.net/pcs/view?xai=AKAOjsv2rihOMLCrL_JYBP4xufy7QmbU8UXT3z6yt2H-8-NAwymeCUTcxJmkY... ## - http://cloud.webtype.com/webtype/ff2/3/c6608520-1978-45ac-9061-74ada664cae4?ec_token=8f7c4c4997246fd7fa920171c994...
You can use [`HARtools::HARviewer`](https://github.com/johndharrison/HARtools/blob/master/R/HARviewer.R) — which this pkg import/exports — to get view the HAR in an interactive HTML widget. You can use [`HARtools::HARviewer`](https://github.com/johndharrison/HARtools/blob/master/R/HARviewer.R) — which this pkg import/exports — to get view the HAR in an interactive HTML widget.
Full web page snapshots are easy-peasy too: Full web page snapshots are easy-peasy too:
``` r ``` r
splash("splash", 8050L) %>% render_png(url = "http://www.marveldirectory.com/individuals/c/captainamerica.htm")
render_png("http://marvel.com/universe/Captain_America_(Steve_Rogers)")
``` ```
![](img/cap.png) ![](img/cap.png)
``` r ``` r
splash("splash", 8050L) %>% render_jpeg(url = "http://static2.comicvine.com/uploads/scale_small/3/31666/5052983-capasr2016001-eptingvar-18bdb.jpg")
render_jpeg("http://marvel.com/universe/Captain_America_(Steve_Rogers)")
``` ```
![](img/cap.jpg) ![](img/cap.jpg)
@ -238,7 +241,7 @@ function main(splash)
end end
' '
res <- splash("localhost") %>% execute_lua(lua_ex) splash_local %>% execute_lua(lua_ex) -> res
rawToChar(res) %>% rawToChar(res) %>%
jsonlite::fromJSON() jsonlite::fromJSON()
@ -250,7 +253,7 @@ rawToChar(res) %>%
### Rendering Widgets ### Rendering Widgets
``` r ``` r
splash_vm <- start_splash(add_tempdir=TRUE) splash_vm <- start_splash(add_tempdir = TRUE)
``` ```
``` r ``` r
@ -266,18 +269,7 @@ DiagrammeR("
") %>% ") %>%
saveWidget("/tmp/diag.html") saveWidget("/tmp/diag.html")
splash("localhost") %>% render_file(url = "/tmp/diag.html", output="html")
render_file("/tmp/diag.html", output="html")
```
## {xml_document}
## <html>
## [1] <head>\n<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">\n<meta charset="utf-8">\n<script src= ...
## [2] <body style="background-color: white; margin: 0px; padding: 40px;">\n<div id="htmlwidget_container">\n<div id="ht ...
``` r
splash("localhost") %>%
render_file("/tmp/diag.html", output="png", wait=2)
``` ```
![](img/diag.png) ![](img/diag.png)
@ -295,7 +287,7 @@ library(testthat)
date() date()
``` ```
## [1] "Tue Feb 15 09:02:35 2017" ## [1] "Tue Feb 21 11:04:49 2017"
``` r ``` r
test_dir("tests/") test_dir("tests/")

BIN
img/cap.jpg

Binary file not shown.

Before

Width:  |  Height:  |  Size: 123 KiB

After

Width:  |  Height:  |  Size: 41 KiB

BIN
img/cap.png

Binary file not shown.

Before

Width:  |  Height:  |  Size: 437 KiB

After

Width:  |  Height:  |  Size: 328 KiB

16
man/install_splash.Rd

@ -1,28 +1,19 @@
% Generated by roxygen2: do not edit by hand % Generated by roxygen2: do not edit by hand
% Please edit documentation in R/docker-splash.r, R/docker.r % Please edit documentation in R/docker-splash.r
\name{install_splash} \name{install_splash}
\alias{install_splash} \alias{install_splash}
\alias{install_splash}
\title{Retrieve the Docker image for Splash} \title{Retrieve the Docker image for Splash}
\usage{ \usage{
install_splash(host = harbor::localhost) install_splash(host = harbor::localhost)
install_splash(host = harbor::localhost)
} }
\arguments{ \arguments{
\item{host}{Docker host; defaults to \code{localhost}} \item{host}{Docker host; defaults to \code{localhost}}
\item{host}{Docker host; defauolts to \code{localhost}}
} }
\value{ \value{
\code{harbor} \code{host} object \code{harbor} \code{host} object
\code{harbor} \code{host} object
} }
\description{ \description{
Retrieve the Docker image for Splash Retrieve the Docker image for Splash
Retrieve the Docker image for Splash
} }
\examples{ \examples{
\dontrun{ \dontrun{
@ -30,9 +21,4 @@ install_splash()
splash_container <- start_splash() splash_container <- start_splash()
stop_splash(splash_container) stop_splash(splash_container)
} }
\dontrun{
install_splash()
splash_container <- start_splash()
stop_splash(splash_container)
}
} }

4
man/render_file.Rd

@ -4,8 +4,8 @@
\alias{render_file} \alias{render_file}
\title{Return the HTML or image (png) of the javascript-rendered page in a local file} \title{Return the HTML or image (png) of the javascript-rendered page in a local file}
\usage{ \usage{
render_file(splash_obj, file_path, output = c("html", "png"), wait = 0, render_file(splash_obj = slpash_local, file_path, output = c("html", "png"),
viewport = "1024x768", ...) wait = 0, viewport = "1024x768", ...)
} }
\arguments{ \arguments{
\item{splash_obj}{Object created by a call to \url{splash}} \item{splash_obj}{Object created by a call to \url{splash}}

9
man/render_har.Rd

@ -4,10 +4,11 @@
\alias{render_har} \alias{render_har}
\title{Return information about Splash interaction with a website in HAR format.} \title{Return information about Splash interaction with a website in HAR format.}
\usage{ \usage{
render_har(splash_obj, url, base_url, response_body = FALSE, timeout = 30, render_har(splash_obj = splash_local, url, base_url, response_body = FALSE,
resource_timeout, wait = 0, proxy, js, js_src, filters, allowed_domains, timeout = 30, resource_timeout, wait = 0, proxy, js, js_src, filters,
allowed_content_types, forbidden_content_types, viewport = "1024x768", allowed_domains, allowed_content_types, forbidden_content_types,
images, headers, body, http_method, save_args, load_args) viewport = "1024x768", images, headers, body, http_method, save_args,
load_args)
} }
\arguments{ \arguments{
\item{splash_obj}{Object created by a call to \link{splash}} \item{splash_obj}{Object created by a call to \link{splash}}

4
man/render_html.Rd

@ -4,8 +4,8 @@
\alias{render_html} \alias{render_html}
\title{Return the HTML of the javascript-rendered page.} \title{Return the HTML of the javascript-rendered page.}
\usage{ \usage{
render_html(splash_obj, url, base_url, timeout = 30, resource_timeout, render_html(splash_obj = splash_local, url, base_url, timeout = 30,
wait = 0, proxy, js, js_src, filters, allowed_domains, resource_timeout, wait = 0, proxy, js, js_src, filters, allowed_domains,
allowed_content_types, forbidden_content_types, viewport = "1024x768", allowed_content_types, forbidden_content_types, viewport = "1024x768",
images, headers, body, http_method, save_args, load_args, raw_html = FALSE) images, headers, body, http_method, save_args, load_args, raw_html = FALSE)
} }

4
man/render_jpeg.Rd

@ -4,8 +4,8 @@
\alias{render_jpeg} \alias{render_jpeg}
\title{Return a image (in JPEG format) of the javascript-rendered page.} \title{Return a image (in JPEG format) of the javascript-rendered page.}
\usage{ \usage{
render_jpeg(splash_obj, url, base_url = NULL, quality = 75, width = 1024, render_jpeg(splash_obj = splash_local, url, base_url = NULL, quality = 75,
height = 768, timeout = 30, resource_timeout, wait = 0, width = 1024, height = 768, timeout = 30, resource_timeout, wait = 0,
render_all = FALSE, proxy, js, js_src, filters, allowed_domains, render_all = FALSE, proxy, js, js_src, filters, allowed_domains,
allowed_content_types, forbidden_content_types, viewport = "1024x768", allowed_content_types, forbidden_content_types, viewport = "1024x768",
images, headers, body, http_method, save_args, load_args) images, headers, body, http_method, save_args, load_args)

4
man/render_json.Rd

@ -4,8 +4,8 @@
\alias{render_json} \alias{render_json}
\title{Return a json-encoded dictionary with information about javascript-rendered webpage.} \title{Return a json-encoded dictionary with information about javascript-rendered webpage.}
\usage{ \usage{
render_json(splash_obj, url, base_url = NULL, quality = 75, width = 1024, render_json(splash_obj = splash_local, url, base_url = NULL, quality = 75,
height = 768, timeout = 30, resource_timeout, wait = 0, width = 1024, height = 768, timeout = 30, resource_timeout, wait = 0,
render_all = FALSE, proxy, js, js_src, filters, allowed_domains, render_all = FALSE, proxy, js, js_src, filters, allowed_domains,
allowed_content_types, forbidden_content_types, viewport = "1024x768", allowed_content_types, forbidden_content_types, viewport = "1024x768",
images, headers, body, http_method, save_args, load_args, html = TRUE, images, headers, body, http_method, save_args, load_args, html = TRUE,

14
man/render_png.Rd

@ -4,11 +4,11 @@
\alias{render_png} \alias{render_png}
\title{Return a image (in PNG format) of the javascript-rendered page.} \title{Return a image (in PNG format) of the javascript-rendered page.}
\usage{ \usage{
render_png(splash_obj, url, base_url, width = 1024, height = 768, render_png(splash_obj = splash_local, url, base_url = NULL, quality = 75,
render_all = TRUE, timeout = 30, resource_timeout, wait = 0, proxy, js, width = 1024, height = 768, timeout = 30, resource_timeout, wait = 0,
js_src, filters, allowed_domains, allowed_content_types, render_all = FALSE, proxy, js, js_src, filters, allowed_domains,
forbidden_content_types, viewport = "1024x768", images, headers, body, allowed_content_types, forbidden_content_types, viewport = "1024x768",
http_method, save_args, load_args) images, headers, body, http_method, save_args, load_args)
} }
\arguments{ \arguments{
\item{splash_obj}{Object created by a call to \link{splash}} \item{splash_obj}{Object created by a call to \link{splash}}
@ -19,14 +19,14 @@ render_png(splash_obj, url, base_url, width = 1024, height = 768,
\item{width, height}{Resize the rendered image to the given width/height (in pixels) keeping the aspect ratio.} \item{width, height}{Resize the rendered image to the given width/height (in pixels) keeping the aspect ratio.}
\item{render_all}{If \code{TRUE} extend the viewport to include the whole webpage (possibly very tall) before rendering.}
\item{timeout}{A timeout (in seconds) for the render (defaults to 30).} \item{timeout}{A timeout (in seconds) for the render (defaults to 30).}
\item{resource_timeout}{A timeout (in seconds) for individual network requests.} \item{resource_timeout}{A timeout (in seconds) for individual network requests.}
\item{wait}{Time (in seconds) to wait for updates after page is loaded (defaults to 0).} \item{wait}{Time (in seconds) to wait for updates after page is loaded (defaults to 0).}
\item{render_all}{If \code{TRUE} extend the viewport to include the whole webpage (possibly very tall) before rendering.}
\item{proxy}{Proxy profile name or proxy URL.} \item{proxy}{Proxy profile name or proxy URL.}
\item{js}{Javascript profile name.} \item{js}{Javascript profile name.}

2
man/splash_active.Rd

@ -4,7 +4,7 @@
\alias{splash_active} \alias{splash_active}
\title{Test if a Splash server is up} \title{Test if a Splash server is up}
\usage{ \usage{
splash_active(splash_obj) splash_active(splash_obj = splash_local)
} }
\arguments{ \arguments{
\item{splash_obj}{A splash connection object} \item{splash_obj}{A splash connection object}

2
man/splash_debug.Rd

@ -5,7 +5,7 @@
\alias{print.splash_debug} \alias{print.splash_debug}
\title{Retrieve debug-level info for a Splash server} \title{Retrieve debug-level info for a Splash server}
\usage{ \usage{
splash_debug(splash_obj) splash_debug(splash_obj = splash_local)
\method{print}{splash_debug}(x, ...) \method{print}{splash_debug}(x, ...)
} }

2
man/splash_history.Rd

@ -4,7 +4,7 @@
\alias{splash_history} \alias{splash_history}
\title{Get information about requests/responses for the pages loaded} \title{Get information about requests/responses for the pages loaded}
\usage{ \usage{
splash_history(splash_obj) splash_history(splash_obj = splash_local)
} }
\arguments{ \arguments{
\item{splash_obj}{A splash connection object} \item{splash_obj}{A splash connection object}

2
man/splash_perf_stats.Rd

@ -4,7 +4,7 @@
\alias{splash_perf_stats} \alias{splash_perf_stats}
\title{Get Splash performance-related statistics} \title{Get Splash performance-related statistics}
\usage{ \usage{
splash_perf_stats(splash_obj) splash_perf_stats(splash_obj = splash_local)
} }
\arguments{ \arguments{
\item{splash_obj}{A splash connection object} \item{splash_obj}{A splash connection object}

2
man/splash_version.Rd

@ -4,7 +4,7 @@
\alias{splash_version} \alias{splash_version}
\title{Get Splash version information} \title{Get Splash version information}
\usage{ \usage{
splash_version(splash_obj) splash_version(splash_obj = splash_local)
} }
\arguments{ \arguments{
\item{splash_obj}{A splash connection object} \item{splash_obj}{A splash connection object}

22
man/start_splash.Rd

@ -1,13 +1,10 @@
% Generated by roxygen2: do not edit by hand % Generated by roxygen2: do not edit by hand
% Please edit documentation in R/docker-splash.r, R/docker.r % Please edit documentation in R/docker-splash.r
\name{start_splash} \name{start_splash}
\alias{start_splash} \alias{start_splash}
\alias{start_splash}
\title{Start a Splash server Docker container} \title{Start a Splash server Docker container}
\usage{ \usage{
start_splash(host = harbor::localhost) start_splash(host = harbor::localhost, add_tempdir = FALSE)
start_splash(host = harbor::localhost)
} }
\arguments{ \arguments{
\item{host}{Docker host; defauolts to `localhost`} \item{host}{Docker host; defauolts to `localhost`}
@ -18,28 +15,18 @@ If `TRUE` then a local temporary directory (made with [tempdir]())
will be added to the mount configuration for use with [render_file](). You will need to will be added to the mount configuration for use with [render_file](). You will need to
ensure the necessary system temp dirs are accessible as a mounts. For ensure the necessary system temp dirs are accessible as a mounts. For
macOS this means adding `/private` to said Docker config.} macOS this means adding `/private` to said Docker config.}
\item{host}{Docker host; defauolts to `localhost`}
} }
\value{ \value{
`harbor` `container` object `harbor` `container` object
`harbor` `container` object
} }
\description{ \description{
If using this in an automation context, you should consider adding a If using this in an automation context, you should consider adding a
`Sys.sleep(3)` (or higher) after starting the docker container. `Sys.sleep(3)` (or higher) after starting the docker container.
Start a Splash server Docker container
} }
\note{ \note{
you need Docker running on your system and have pulled the container with you need Docker running on your system and have pulled the container with
[install_spash] for this to work. You should save the resultant `host` [install_spash] for this to work. You should save the resultant `host`
object for use in [stop_splash]. object for use in [stop_splash].
you need Docker running on your system and have pulled the container with
[install_spash] for this to work. You should save the resultant `host`
object for use in [stop_splash].
} }
\examples{ \examples{
\dontrun{ \dontrun{
@ -47,9 +34,4 @@ install_splash()
splash_container <- start_splash() splash_container <- start_splash()
stop_splash(splash_container) stop_splash(splash_container)
} }
\dontrun{
install_splash()
splash_container <- start_splash()
stop_splash(splash_container)
}
} }

20
man/stop_splash.Rd

@ -1,36 +1,23 @@
% Generated by roxygen2: do not edit by hand % Generated by roxygen2: do not edit by hand
% Please edit documentation in R/docker-splash.r, R/docker.r % Please edit documentation in R/docker-splash.r
\name{stop_splash} \name{stop_splash}
\alias{stop_splash} \alias{stop_splash}
\alias{stop_splash}
\title{Stop a running a Splash server Docker container} \title{Stop a running a Splash server Docker container}
\usage{ \usage{
stop_splash(splash_container) stop_splash(splash_container)
stop_splash(splash_container)
} }
\arguments{ \arguments{
\item{splash_container}{saved Splash container id from [start_splash]} \item{splash_container}{saved Splash container id from [start_splash]}
\item{splash_container}{Docker `container` object created by [start_splash]} \item{splash_container}{Docker `container` object created by [start_splash]}
\item{splash_container}{saved Splash container id from [start_splash]}
\item{splash_container}{Docker `container` object created by [start_splash]}
} }
\description{ \description{
Stop a running a Splash server Docker container Stop a running a Splash server Docker container
Stop a running a Splash server Docker container
} }
\note{ \note{
you need Docker running on your system and have pulled the container with you need Docker running on your system and have pulled the container with
[install_spash] and started the Splash container with [start_splash] for this [install_spash] and started the Splash container with [start_splash] for this
to work. You will need the `container` object from [start_splash] for this to work. to work. You will need the `container` object from [start_splash] for this to work.
you need Docker running on your system and have pulled the container with
[install_spash] and started the Splash container with [start_splash] for this
to work. You will need the `container` object from [start_splash] for this to work.
} }
\examples{ \examples{
\dontrun{ \dontrun{
@ -38,9 +25,4 @@ install_splash()
splash_container <- start_splash() splash_container <- start_splash()
stop_splash(splash_container) stop_splash(splash_container)
} }
\dontrun{
install_splash()
splash_container <- start_splash()
stop_splash(splash_container)
}
} }

Loading…
Cancel
Save