Browse Source

on the road to 7.0

master
boB Rudis 4 years ago
parent
commit
b548840f6e
No known key found for this signature in database GPG Key ID: 1D7529BE14E2BBA9
  1. 6
      DESCRIPTION
  2. 7
      NAMESPACE
  3. 7
      NEWS.md
  4. 160
      R/as-data-frame-har.R
  5. 15
      R/check-or-report-status.R
  6. 20
      R/render-har.r
  7. 27
      R/render-html.r
  8. 28
      R/render-jpg.r
  9. 26
      R/render-png.r
  10. 4
      R/utils-infix-helpers.R
  11. 9
      README.Rmd
  12. 328
      README.md
  13. BIN
      img/cap.jpg
  14. BIN
      img/cap.png
  15. 34
      man/as_data_frame.harentry.Rd
  16. 21
      man/execute_lua.Rd
  17. 18
      man/get_content_size.Rd
  18. 18
      man/get_content_type.Rd
  19. 18
      man/get_har_entry.Rd
  20. 18
      man/get_header_val.Rd
  21. 19
      man/get_headers.Rd
  22. 18
      man/get_request_type.Rd
  23. 18
      man/get_request_url.Rd
  24. 18
      man/get_response_body.Rd
  25. 18
      man/get_response_url.Rd
  26. 18
      man/har_entry_count.Rd
  27. 5
      man/install_splash.Rd
  28. 50
      man/render_har.Rd
  29. 50
      man/render_html.Rd
  30. 54
      man/render_jpeg.Rd
  31. 58
      man/render_json.Rd
  32. 52
      man/render_png.Rd
  33. 9
      man/splash_active.Rd
  34. 20
      man/splash_add_lua.Rd
  35. 20
      man/splash_click.Rd
  36. 9
      man/splash_debug.Rd
  37. 9
      man/splash_enable_javascript.Rd
  38. 20
      man/splash_focus.Rd
  39. 20
      man/splash_go.Rd
  40. 20
      man/splash_har.Rd
  41. 20
      man/splash_har_reset.Rd
  42. 9
      man/splash_history.Rd
  43. 20
      man/splash_html.Rd
  44. 9
      man/splash_images.Rd
  45. 8
      man/splash_perf_stats.Rd
  46. 9
      man/splash_plugins.Rd
  47. 20
      man/splash_png.Rd
  48. 20
      man/splash_press.Rd
  49. 9
      man/splash_private_mode.Rd
  50. 20
      man/splash_release.Rd
  51. 9
      man/splash_response_body.Rd
  52. 20
      man/splash_send_keys.Rd
  53. 20
      man/splash_send_text.Rd
  54. 2
      man/splash_user_agent.Rd
  55. 8
      man/splash_version.Rd
  56. 21
      man/splash_wait.Rd
  57. 1
      man/splashr.Rd
  58. 8
      man/start_splash.Rd
  59. 5
      man/stop_splash.Rd
  60. 19
      man/tidy_har.Rd

6
DESCRIPTION

@ -1,8 +1,8 @@
Package: splashr
Type: Package
Title: Tools to Work with the 'Splash' 'JavaScript' Rendering and Scraping Service
Version: 0.6.0
Date: 2019-02-24
Version: 0.7.0
Date: 2020-02-18
Encoding: UTF-8
Authors@R: c(
person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"),
@ -46,5 +46,5 @@ Imports:
HARtools,
jsonlite,
lubridate
RoxygenNote: 6.1.1
RoxygenNote: 7.0.2
VignetteBuilder: knitr

7
NAMESPACE

@ -1,17 +1,11 @@
# Generated by roxygen2: do not edit by hand
S3method(as.data.frame,har)
S3method(as.data.frame,harentries)
S3method(as.data.frame,harentry)
S3method(print,splash_debug)
S3method(print,splash_json)
S3method(print,splashr)
export("%>%")
export(HARviewer)
export(HARviewerOutput)
export(as_data_frame.har)
export(as_data_frame.harentries)
export(as_data_frame.harentry)
export(as_har)
export(as_httr_req)
export(as_response)
@ -81,6 +75,7 @@ export(splash_version)
export(splash_wait)
export(start_splash)
export(stop_splash)
export(tidy_har)
export(ua_android_samsung)
export(ua_apple_tv)
export(ua_chromecast)

7
NEWS.md

@ -1,3 +1,10 @@
0.7.0
* Changed `as[._]data[._]frame()` function to `tidy_har()`
* Improved API error messages
* Added support for `http2` (<https://splash.readthedocs.io/en/stable/changes.html#id1>)
* Added support for the alpha/experimental Splash chromium engine
(<https://splash.readthedocs.io/en/stable/changes.html#id2>)
0.6.0
* Switch Docker orchestration to the `stevedore` package

160
R/as-data-frame-har.R

@ -1,62 +1,116 @@
#' Turns a "HAR"-like object into a data frame(tibble)
#'
#' @md
#' @param x A `harentry` object
#' @param ... ignored
#' @return data frame (tibble)
#' @export
as_data_frame.harentry <- function(x, ...) {
req <- x$request
resp <- x$response
data_frame(
request_url = req$url,
request_method = req$method,
request_http_version = req$httpVersion,
request_query_string = list(req$queryString),
request_header_size = req$headersSize,
requestheaders = list(if (length(req$headers) > 0) flatten_df(req$headers) else data_frame()),
requestcookies = list(if (length(req$cookies) > 0) flatten_df(req$cookies) else data_frame()),
response_url = resp$url,
response_http_version = resp$httpVersion,
status_text = resp$statusText,
status = resp$status,
ok = resp$ok,
redirect_url = resp$redirectURL,
response_headers_size = resp$headersSize,
response_headers = list(if (length(resp$headers) > 0) flatten_df(resp$headers) else data_frame()),
response_cookies = list(if (length(resp$cookies) > 0) flatten_df(resp$cookies) else data_frame()),
response_body_size = resp$bodySize,
content_type = resp$content$mimeType,
content_encoding = resp$content$encoding %||% NA_character_,
content_size = resp$content$size,
content = resp$content$text %||% NA_character_
)
.tidy_one_entry <- function(x, include_content = TRUE) {
}
.x <- x
#' @rdname as_data_frame.harentry
#' @export
as_data_frame.harentries <- function(x, ...) {
map_df(x, as_data_frame)
}
if (length(.x[["timings"]])) {
data.frame(
stage = names(.x[["timings"]]),
value = unlist(.x[["timings"]], use.names = FALSE),
stringsAsFactors = FALSE
) -> timings
} else {
timings <- data.frame(stringsAsFactors = FALSE)
}
class(timings) <- c("tbl_df", "tbl", "data.frame")
if (length(.x[["request"]][["headers"]])) {
data.frame(
name =vapply(.x[["request"]][["headers"]], `[[`, character(1), "name", USE.NAMES = FALSE),
value = vapply(.x[["request"]][["headers"]], `[[`, character(1), "value", USE.NAMES = FALSE),
stringsAsFactors = FALSE
) -> req_headers
} else {
req_headers <- data.frame(stringsAsFactors = FALSE)
}
class(req_headers) <- c("tbl_df", "tbl", "data.frame")
if (length(.x[["response"]][["headers"]])) {
data.frame(
name =vapply(.x[["response"]][["headers"]], `[[`, character(1), "name", USE.NAMES = FALSE),
value = vapply(.x[["response"]][["headers"]], `[[`, character(1), "value", USE.NAMES = FALSE),
stringsAsFactors = FALSE
) -> headers
} else {
headers <- data.frame(stringsAsFactors = FALSE)
}
class(headers) <- c("tbl_df", "tbl", "data.frame")
if (length(.x[["request"]][["cookies"]])) {
data.frame(
name =vapply(.x[["request"]][["cookies"]], `[[`, character(1), "name", USE.NAMES = FALSE),
value = vapply(.x[["request"]][["cookies"]], `[[`, character(1), "value", USE.NAMES = FALSE),
stringsAsFactors = FALSE
) -> req_cookies
} else {
req_cookies <- data.frame(stringsAsFactors = FALSE)
}
class(req_cookies) <- c("tbl_df", "tbl", "data.frame")
if (length(.x[["response"]][["cookies"]])) {
data.frame(
name =vapply(.x[["response"]][["cookies"]], `[[`, character(1), "name", USE.NAMES = FALSE),
value = vapply(.x[["response"]][["cookies"]], `[[`, character(1), "value", USE.NAMES = FALSE),
stringsAsFactors = FALSE
) -> res_cookies
} else {
res_cookies <- data.frame(stringsAsFactors = FALSE)
}
class(res_cookies) <- c("tbl_df", "tbl", "data.frame")
data.frame(
status = .x[["response"]][["status"]] %l0% NA_character_,
started = .x[["startedDateTime"]] %l0% NA_character_,
total_time = .x[["time"]] %l0% NA_integer_,
page_ref = .x[["pageref"]] %l0% NA_character_,
timings = I(list(timings)),
req_url = .x[["request"]][["url"]] %l0% NA_character_,
req_method = .x[["request"]][["method"]] %l0% NA_character_,
req_http_version = .x[["request"]][["httpVersion"]] %l0% NA_character_,
req_hdr_size = .x[["request"]][["headersSize"]] %l0% NA_character_,
req_headers = I(list(req_headers)),
req_cookies = I(list(req_cookies)),
resp_url = .x[["response"]][["url"]] %l0% NA_character_,
resp_rdrurl = .x[["response"]][["redirectURL"]] %l0% NA_character_,
resp_type = .x[["response"]][["content"]][["mimeType"]] %l0% NA_character_,
resp_size = .x[["resonse"]][["bodySize"]] %l0% NA_integer_,
resp_cookies = I(list(res_cookies)),
resp_headers = I(list(headers)),
resp_encoding = .x[["resonse"]][["content"]][["encoding"]] %l0% NA_character_,
resp_content_size = as.numeric(.x[["resonse"]][["content"]][["size"]]) %l0% NA_real_,
stringsAsFactors = FALSE
) -> out
if (include_content) out$resp_content <- .x[["resonse"]][["content"]][["text"]] %l0% NA_character_
class(out) <- c("tbl_df", "tbl", "data.frame")
out
#' @rdname as_data_frame.harentry
#' @export
as_data_frame.har <- function(x, ...) {
as_data_frame(x$log$entries)
}
#' Turn a gnHARly HAR object into a tidy data frame (tibble)
#'
#' @md
#' @param x A `harentry` object
#' @param include_content if `TRUE` (the default) the encoded element content will be returned in the data frame
#' @return data frame (tibble)
#' @export
#' @rdname as_data_frame.harentry
as.data.frame.har <- as_data_frame.har
tidy_har <- function(.x, include_content = TRUE) {
#' @export
#' @rdname as_data_frame.harentry
as.data.frame.harentries <- as_data_frame.harentries
if (inherits(.x, "har")) {
out <- tidy_har(.x[["log"]][["entries"]], include_content = include_content)
} else if (inherits(.x, "harlog")) {
out <- tidy_har(.x[["entries"]], include_content = include_content)
} else if (inherits(.x, "harentries")) {
out <- do.call(rbind.data.frame, lapply(.x, .tidy_one_entry, include_content = include_content))
class(out) <- c("tbl_df", "tbl", "data.frame")
} else if (inherits(.x, "harentry")) {
out <- .tidy_one_entry(.x, include_content = include_content)
} else {
stopifnot(inherits(.x, "harentries"))
}
#' @export
#' @rdname as_data_frame.harentry
as.data.frame.harentry <- as_data_frame.harentry
out
}

15
R/check-or-report-status.R

@ -0,0 +1,15 @@
check_or_report_status <- function(res) {
if (httr::status_code(res) >= 400) {
msg <- httr::content(res, as="text", encoding="UTF-8")
msg <- jsonlite::fromJSON(msg)
httr::stop_for_status(
x = res,
task = sprintf(
fmt = "render due to %s",
paste0(c(msg[["type"]], msg[["description"]], msg[["info"]]), collapse="; ")
)
)
}
}

20
R/render-har.r

@ -13,12 +13,24 @@
render_har <- function(splash_obj = splash_local, url, base_url, response_body=FALSE, timeout=30, resource_timeout, wait=0,
proxy, js, js_src, filters, allowed_domains, allowed_content_types,
forbidden_content_types, viewport="1024x768", images, headers, body,
http_method, save_args, load_args) {
http_method, save_args, load_args, http2 = FALSE,
engine = c("webkit", "chromium")) {
wait <- check_wait(wait)
params <- list(url=url, timeout=timeout, wait=wait, viewport=jsonlite::unbox(viewport),
response_body=as.numeric(response_body))
engine <- match.arg(engine[1], c("webkit", "chromium"))
http2 <- ifelse(engine == "chromium", 1, as.integer(as.logical(http2[1])))
list(
url = url,
timeout = timeout,
wait = wait,
viewport = jsonlite::unbox(viewport),
response_body = as.numeric(response_body),
http2 = http2,
engine = engine
) -> params
if (!missing(base_url)) params$base_url <- jsonlite::unbox(base_url)
if (!missing(resource_timeout)) params$resource_timeout <- resource_timeout
@ -45,7 +57,7 @@ render_har <- function(splash_obj = splash_local, url, base_url, response_body=F
)
}
httr::stop_for_status(res)
check_or_report_status(res)
out <- httr::content(res, as="text", encoding="UTF-8")
spl <- jsonlite::fromJSON(out, flatten=FALSE, simplifyVector=FALSE)

27
R/render-html.r

@ -5,9 +5,9 @@
#' @md
#' @param splash_obj Object created by a call to [splash()]
#' @param url The URL to render (required)
#' @param base_url The base url to render the page with.
#' @param base_url The base URL to render the page with.
#' @param timeout A timeout (in seconds) for the render (defaults to 30). Without
#' reconfiguring the startup parameters of the Splash server (not this package)
#' re-configuring the start-up parameters of the Splash server (not this package)
#' the maximum allowed value for the timeout is 60 seconds.
#' @param resource_timeout A timeout (in seconds) for individual network requests.
#' @param wait Time (in seconds) to wait for updates after page is loaded (defaults to 0).
@ -25,13 +25,16 @@
#' present, Splash will abort any request if the response’s content type matches
#' any of the content types in this list. Wildcards are supported.
#' @param viewport View width and height (in pixels) of the browser viewport to render the
#' web page. Format is “<width>x<height>”, e.g. 800x600. Default value is "full".
#' web page. Format is “width>xheight”, e.g. 800x600. Default value is "full".
#' @param images Whether to download images.
#' @param headers HTTP headers to set for the first outgoing request.
#' @param body Body of HTTP POST request to be sent if method is POST.
#' @param http_method HTTP method of outgoing Splash request.
#' @param save_args A list of argument names to put in cache.
#' @param load_args Parameter values to load from cache
#' @param http2 Enable or disable HTTP2 support. `TRUE` to enable; `FALSE` to disable; defaults to `FALSE`
#' when `engine` is `webkit` due to malformed behaviour in 3.4.x of Splash
#' @param engine one of `webkit` or `chromium`; defaults to `webkit`
#' @param raw_html if `TRUE` then return a character vector vs an XML document. Only valid for `render_html`
#' @family splash_renderers
#' @return An XML document. Note that this is processed by [xml2::read_html()] so it will not be
@ -43,11 +46,23 @@
render_html <- function(splash_obj = splash_local, url, base_url, timeout=30, resource_timeout, wait=0,
proxy, js, js_src, filters, allowed_domains, allowed_content_types,
forbidden_content_types, viewport="1024x768", images, headers, body,
http_method, save_args, load_args, raw_html=FALSE) {
http_method, save_args, load_args, http2 = FALSE,
engine = c("webkit", "chromium"), raw_html=FALSE) {
wait <- check_wait(wait)
params <- list(url=url, timeout=timeout, wait=wait, viewport=jsonlite::unbox(viewport))
engine <- match.arg(engine[1], c("webkit", "chromium"))
http2 <- ifelse(engine == "chromium", 1, as.integer(as.logical(http2[1])))
list(
url = url,
timeout = timeout,
wait = wait,
viewport = jsonlite::unbox(viewport),
http2 = http2,
engine = engine
) -> params
if (!missing(base_url)) params$base_url <- jsonlite::unbox(base_url)
if (!missing(resource_timeout)) params$resource_timeout <- resource_timeout
@ -74,7 +89,7 @@ render_html <- function(splash_obj = splash_local, url, base_url, timeout=30, re
)
}
httr::stop_for_status(res)
check_or_report_status(res)
out <- httr::content(res, as="text", encoding="UTF-8")

28
R/render-jpg.r

@ -13,15 +13,29 @@ render_jpeg <- render_jpg <- function(
timeout=30, resource_timeout, wait=0, render_all=TRUE,
proxy, js, js_src, filters, allowed_domains, allowed_content_types,
forbidden_content_types, viewport="full", images, headers, body,
http_method, save_args, load_args) {
http_method, save_args, load_args, http2 = FALSE,
engine = c("webkit", "chromium")) {
wait <- check_wait(wait)
params <- list(url=url, timeout=timeout,
wait=if (render_all & wait == 0) 0.5 else wait,
viewport=viewport,
quality=quality,
render_all=as.numeric(render_all))
engine <- match.arg(engine[1], c("webkit", "chromium"))
http2 <- if (engine == "chromium") NULL else as.integer(as.logical(http2[1]))
viewport <- if (engine == "chromium") NULL else jsonlite::unbox(viewport[1])
render_all <- if (engine == "chromium") NULL else as.integer(render_all[1])
wait <- if (is.null(render_all)) 0.5 else if (render_all & wait == 0) 0.5 else wait
list(
url = url,
timeout = timeout,
wait = wait,
viewport = viewport,
render_all = render_all,
quality = quality,
http2 = http2,
engine = engine
) -> params
if (!missing(width)) params$width <- width
if (!missing(height)) params$height <- height
@ -51,7 +65,7 @@ render_jpeg <- render_jpg <- function(
)
}
httr::stop_for_status(res)
check_or_report_status(res)
magick::image_read(httr::content(res, as="raw"))

26
R/render-png.r

@ -18,15 +18,27 @@ render_png <- function(
timeout=30, resource_timeout, wait=0, render_all=TRUE,
proxy, js, js_src, filters, allowed_domains, allowed_content_types,
forbidden_content_types, viewport="full", images, headers, body,
http_method, save_args, load_args) {
http_method, save_args, load_args, http2 = FALSE,
engine = c("webkit", "chromium")) {
wait <- check_wait(wait)
params <- list(
engine <- match.arg(engine[1], c("webkit", "chromium"))
http2 <- if (engine == "chromium") NULL else as.integer(as.logical(http2[1]))
viewport <- if (engine == "chromium") NULL else jsonlite::unbox(viewport[1])
render_all <- if (engine == "chromium") NULL else as.integer(render_all[1])
wait <- if (is.null(render_all)) 0.5 else if (render_all & wait == 0) 0.5 else wait
list(
url = url, timeout = timeout,
wait = if (render_all & wait == 0) 0.5 else wait,
viewport = jsonlite::unbox(viewport),
render_all = as.numeric(render_all)
)
wait = wait,
viewport = viewport,
render_all = render_all,
http2 = http2,
engine = engine
) -> params
if (!missing(width)) params$width <- width
if (!missing(height)) params$height <- height
@ -56,7 +68,7 @@ render_png <- function(
)
}
httr::stop_for_status(res)
check_or_report_status(res)
magick::image_read(httr::content(res, as = "raw"))
}

4
R/utils-infix-helpers.R

@ -0,0 +1,4 @@
`%l0%` <- function(x, y) if (length(x) == 0) y else x
`%||%` <- function(x, y) if (is.null(x)) y else x
`%@%` <- function(x, name) attr(x, name, exact = TRUE)
`%nin%` <- function(x, table) match(x, table, nomatch = 0) == 0

9
README.Rmd

@ -16,8 +16,8 @@ It's also an alternative to the somewhat abandoned `phantomjs` (which you can us
The package uses the [`stevedore`](https://github.com/richfitz/stevedore) package to orchestrate Docker on your system (if you have Docker and more on how to use the `stevedore` integration below) but you can also do get it running in Docker on the command-line with two commands:
sudo docker pull scrapinghub/splash:3.0
sudo docker run -p 5023:5023 -p 8050:8050 -p 8051:8051 scrapinghub/splash:3.0
sudo docker pull scrapinghub/splash:latest --disable-browser-caches
sudo docker run -p 5023:5023 -p 8050:8050 -p 8051:8051 scrapinghub/splash:latest --disable-browser-caches
Do whatever you Windows ppl do with Docker on your systems to make ^^ work.
@ -37,7 +37,7 @@ and then run:
stop_splash(splash_container)
when done. All of that happens on your localhost and you will not need to specify `splash_obj` to many of the `splashr` functions if you're running Splash in this default configuration as long as you use named parameters. You can also use the pre-defined `splash_local` object if you want to use positional parameters.
when done. All of that happens on your `localhost` and you will not need to specify `splash_obj` to many of the `splashr` functions if you're running Splash in this default configuration as long as you use named parameters. You can also use the pre-defined `splash_local` object if you want to use positional parameters.
Now, you can run Selenium in Docker, so this is not unique to Splash. But, a Docker context makes it so that you don't have to run or maintain icky Python stuff directly on your system. Leave it in the abandoned warehouse district where it belongs.
@ -84,6 +84,7 @@ Mini-DSL (domain-specific language). These can be used to create a "script" with
Helpers:
- `tidy_har`: Turn a gnHARly HAR object into a tidy data frame
- `get_body_size`: Retrieve size of content | body | headers
- `get_content_sie`: Retrieve size of content | body | headers
- `get_content_type` Retrieve or test content type of a HAR request object
@ -164,6 +165,8 @@ You can also profile pages:
render_har(url = "http://www.poynter.org/") -> har
print(har)
tidy_har(har)
```
You can use [`HARtools::HARviewer`](https://github.com/johndharrison/HARtools/blob/master/R/HARviewer.R) — which this pkg import/exports — to get view the HAR in an interactive HTML widget.

328
README.md

@ -1,29 +1,53 @@
[![Travis-CI Build Status](https://travis-ci.org/hrbrmstr/splashr.svg?branch=master)](https://travis-ci.org/hrbrmstr/splashr) [![Coverage Status](https://img.shields.io/codecov/c/github/hrbrmstr/splashr/master.svg)](https://codecov.io/github/hrbrmstr/splashr?branch=master) [![CRAN\_Status\_Badge](http://www.r-pkg.org/badges/version/splashr)](https://cran.r-project.org/package=splashr) [![](http://cranlogs.r-pkg.org/badges/splashr)](http://cran.rstudio.com/web/packages/splashr/index.html)
# `splashr` : Tools to Work with the 'Splash' JavaScript Rendering Service
TL;DR: This package works with Splash rendering servers which are really just a REST API & `lua` scripting interface to a QT browser. It's an alternative to the Selenium ecosystem which was really engineered for application testing & validation.
Sometimes, all you need is a page scrape after javascript has been allowed to roam wild and free over meticulously crafted HTML tags. So, this package does not do *everything* Selenium can in pure R (though, the Lua interface is equally as powerful and accessible via R), but if you're just trying to get a page back that needs javascript rendering, this is a nice, lightweight, consistent alternative.
It's also an alternative to the somewhat abandoned `phantomjs` (which you can use in R within or without a Selenium context as it's it's own webdriver) and it may be useful to compare renderings between this package & `phantomjs`.
The package uses the [`stevedore`](https://github.com/richfitz/stevedore) package to orchestrate Docker on your system (if you have Docker and more on how to use the `stevedore` integration below) but you can also do get it running in Docker on the command-line with two commands:
sudo docker pull scrapinghub/splash:3.0
sudo docker run -p 5023:5023 -p 8050:8050 -p 8051:8051 scrapinghub/splash:3.0
Do whatever you Windows ppl do with Docker on your systems to make ^^ work.
Folks super-new to Docker on Unix-ish platforms should [make sure to do](https://github.com/hrbrmstr/splashr/issues/3#issuecomment-280686494):
[![Travis-CI Build
Status](https://travis-ci.org/hrbrmstr/splashr.svg?branch=master)](https://travis-ci.org/hrbrmstr/splashr)
[![Coverage
Status](https://img.shields.io/codecov/c/github/hrbrmstr/splashr/master.svg)](https://codecov.io/github/hrbrmstr/splashr?branch=master)
[![CRAN\_Status\_Badge](http://www.r-pkg.org/badges/version/splashr)](https://cran.r-project.org/package=splashr)
[![](http://cranlogs.r-pkg.org/badges/splashr)](http://cran.rstudio.com/web/packages/splashr/index.html)
# `splashr` : Tools to Work with the ‘Splash’ JavaScript Rendering Service
TL;DR: This package works with Splash rendering servers which are really
just a REST API & `lua` scripting interface to a QT browser. It’s an
alternative to the Selenium ecosystem which was really engineered for
application testing & validation.
Sometimes, all you need is a page scrape after javascript has been
allowed to roam wild and free over meticulously crafted HTML tags. So,
this package does not do *everything* Selenium can in pure R (though,
the Lua interface is equally as powerful and accessible via R), but if
you’re just trying to get a page back that needs javascript rendering,
this is a nice, lightweight, consistent alternative.
It’s also an alternative to the somewhat abandoned `phantomjs` (which
you can use in R within or without a Selenium context as it’s it’s own
webdriver) and it may be useful to compare renderings between this
package & `phantomjs`.
The package uses the
[`stevedore`](https://github.com/richfitz/stevedore) package to
orchestrate Docker on your system (if you have Docker and more on how to
use the `stevedore` integration below) but you can also do get it
running in Docker on the command-line with two commands:
sudo docker pull scrapinghub/splash:latest --disable-browser-caches
sudo docker run -p 5023:5023 -p 8050:8050 -p 8051:8051 scrapinghub/splash:latest --disable-browser-caches
Do whatever you Windows ppl do with Docker on your systems to make ^^
work.
Folks super-new to Docker on Unix-ish platforms should [make sure to
do](https://github.com/hrbrmstr/splashr/issues/3#issuecomment-280686494):
sudo groupadd docker
sudo usermod -aG docker $USER
(`$USER` is your username and shld be defined for you in the environment)
(`$USER` is your username and shld be defined for you in the
environment)
If using the [`stevedore`](https://github.com/richfitz/stevedore) package you can use the convience wrappers in this pacakge:
If using the [`stevedore`](https://github.com/richfitz/stevedore)
package you can use the convience wrappers in this pacakge:
install_splash()
splash_container <- start_splash()
@ -32,84 +56,131 @@ and then run:
stop_splash(splash_container)
when done. All of that happens on your localhost and you will not need to specify `splash_obj` to many of the `splashr` functions if you're running Splash in this default configuration as long as you use named parameters. You can also use the pre-defined `splash_local` object if you want to use positional parameters.
when done. All of that happens on your `localhost` and you will not need
to specify `splash_obj` to many of the `splashr` functions if you’re
running Splash in this default configuration as long as you use named
parameters. You can also use the pre-defined `splash_local` object if
you want to use positional parameters.
Now, you can run Selenium in Docker, so this is not unique to Splash. But, a Docker context makes it so that you don't have to run or maintain icky Python stuff directly on your system. Leave it in the abandoned warehouse district where it belongs.
Now, you can run Selenium in Docker, so this is not unique to Splash.
But, a Docker context makes it so that you don’t have to run or maintain
icky Python stuff directly on your system. Leave it in the abandoned
warehouse district where it belongs.
All you need for this package to work is a running Splash instance. You provide the host/port for it and it's scrape-tastic fun from there!
All you need for this package to work is a running Splash instance. You
provide the host/port for it and it’s scrape-tastic fun from there\!
### About Splash
> 'Splash' <https://github.com/scrapinghub/splash> is a javascript rendering service. It’s a lightweight web browser with an 'HTTP' API, implemented in Python using 'Twisted'and 'QT' \[and provides some of the core functionality of the 'RSelenium' or 'seleniumPipes' R packages but with a Java-free footprint\]. The (twisted) 'QT' reactor is used to make the sever fully asynchronous allowing to take advantage of 'webkit' concurrency via QT main loop. Some of Splash features include the ability to process multiple webpages in parallel; retrieving HTML results and/or take screenshots; disabling images or use Adblock Plus rules to make rendering faster; executing custom JavaScript in page context; getting detailed rendering info in HAR format.
> ‘Splash’ <https://github.com/scrapinghub/splash> is a javascript
> rendering service. It’s a lightweight web browser with an ‘HTTP’ API,
> implemented in Python using ‘Twisted’and ’QT’ \[and provides some of
> the core functionality of the ‘RSelenium’ or ‘seleniumPipes’ R
> packages but with a Java-free footprint\]. The (twisted) ‘QT’ reactor
> is used to make the sever fully asynchronous allowing to take
> advantage of ‘webkit’ concurrency via QT main loop. Some of Splash
> features include the ability to process multiple webpages in parallel;
> retrieving HTML results and/or take screenshots; disabling images or
> use Adblock Plus rules to make rendering faster; executing custom
> JavaScript in page context; getting detailed rendering info in HAR
> format.
The following functions are implemented:
- `render_html`: Return the HTML of the javascript-rendered page.
- `render_har`: Return information about Splash interaction with a website in [HAR](http://www.softwareishard.com/blog/har-12-spec/) format.
- `render_jpeg`: Return a image (in JPEG format) of the javascript-rendered page.
- `render_png`: Return a image (in PNG format) of the javascript-rendered page.
- `execute_lua`: Execute a custom rendering script and return a result.
- `splash`: Configure parameters for connecting to a Splash server
- `install_splash`: Retrieve the Docker image for Splash
- `start_splash`: Start a Splash server Docker container
- `stop_splash`: Stop a running a Splash server Docker container
Mini-DSL (domain-specific language). These can be used to create a "script" without actually scripting in Lua. They are a less-powerful/configurable set of calls than what you can make with a full Lua function but the idea is to have it take care of very common but simple use-cases, like waiting a period of time before capturing a HAR/HTML/PNG image of a site:
- `splash_plugins`: Enable or disable browser plugins (e.g. Flash).
- `splash_click`: Trigger mouse click event in web page.
- `splash_focus`: Focus on a document element provided by a CSS selector
- `splash_images`: Enable/disable images
- `splash_response_body`: Enable or disable response content tracking.
- `splash_go`: Go to an URL.
- `splash_wait`: Wait for a period time
- `splash_har`: Return information about Splash interaction with a website in HAR format.
- `splash_html`: Return a HTML snapshot of a current page.
- `splash_png`: Return a screenshot of a current page in PNG format.
- `splash_press`: Trigger mouse press event in web page.
- `splash_release`: Trigger mouse release event in web page.
- `splash_send_keys`: Send keyboard events to page context.
- `splash_send_text`: Send text as input to page context, literally, character by character.
- `splash_user_agent`: Overwrite the User-Agent header for all further requests. NOTE: There are many "helper" user agent strings to go with `splash_user_agent`. Look for objects in `splashr` starting with `ua_`.
`httr` helpers. These help turn various bits of `splashr` objects into `httr`-ish things:
- `as_req`: Turn a HAR response entry into a working `httr` function you can use to make a request with
- `as_request`: Turn a HAR response entry into an `httr` `response`-like object (i.e. you can use `httr::content()` on it)
- `render_html`: Return the HTML of the javascript-rendered page.
- `render_har`: Return information about Splash interaction with a
website in [HAR](http://www.softwareishard.com/blog/har-12-spec/)
format.
- `render_jpeg`: Return a image (in JPEG format) of the
javascript-rendered page.
- `render_png`: Return a image (in PNG format) of the
javascript-rendered page.
- `execute_lua`: Execute a custom rendering script and return a
result.
- `splash`: Configure parameters for connecting to a Splash server
- `install_splash`: Retrieve the Docker image for Splash
- `start_splash`: Start a Splash server Docker container
- `stop_splash`: Stop a running a Splash server Docker container
Mini-DSL (domain-specific language). These can be used to create a
“script” without actually scripting in Lua. They are a
less-powerful/configurable set of calls than what you can make with a
full Lua function but the idea is to have it take care of very common
but simple use-cases, like waiting a period of time before capturing a
HAR/HTML/PNG image of a site:
- `splash_plugins`: Enable or disable browser plugins (e.g. Flash).
- `splash_click`: Trigger mouse click event in web page.
- `splash_focus`: Focus on a document element provided by a CSS
selector
- `splash_images`: Enable/disable images
- `splash_response_body`: Enable or disable response content tracking.
- `splash_go`: Go to an URL.
- `splash_wait`: Wait for a period time
- `splash_har`: Return information about Splash interaction with a
website in HAR format.
- `splash_html`: Return a HTML snapshot of a current page.
- `splash_png`: Return a screenshot of a current page in PNG format.
- `splash_press`: Trigger mouse press event in web page.
- `splash_release`: Trigger mouse release event in web page.
- `splash_send_keys`: Send keyboard events to page context.
- `splash_send_text`: Send text as input to page context, literally,
character by character.
- `splash_user_agent`: Overwrite the User-Agent header for all further
requests. NOTE: There are many “helper” user agent strings to go
with `splash_user_agent`. Look for objects in `splashr` starting
with `ua_`.
`httr` helpers. These help turn various bits of `splashr` objects into
`httr`-ish things:
- `as_req`: Turn a HAR response entry into a working `httr` function
you can use to make a request with
- `as_request`: Turn a HAR response entry into an `httr`
`response`-like object (i.e. you can use `httr::content()` on it)
Helpers:
- `get_body_size`: Retrieve size of content | body | headers
- `get_content_sie`: Retrieve size of content | body | headers
- `get_content_type` Retrieve or test content type of a HAR request object
- `get_headers_size` Retrieve size of content | body | headers
- `is_binary`: Retrieve or test content type of a HAR request object
- `is_content_type`: Retrieve or test content type of a HAR request object
- `is_css`: Retrieve or test content type of a HAR request object
- `is_gif`: Retrieve or test content type of a HAR request object
- `is_html`: Retrieve or test content type of a HAR request object
- `is_javascript`: Retrieve or test content type of a HAR request object
- `is_jpeg`: Retrieve or test content type of a HAR request object
- `is_json`: Retrieve or test content type of a HAR request object
- `is_plain`: Retrieve or test content type of a HAR request object
- `is_png`: Retrieve or test content type of a HAR request object
- `is_svg`: Retrieve or test content type of a HAR request object
- `is_xhr`: Retrieve or test content type of a HAR request object
- `is_xml`: Retrieve or test content type of a HAR request object
Some functions from `HARtools` are imported/exported and `%>%` is imported/exported.
- `tidy_har`: Turn a gnHARly HAR object into a tidy data frame
- `get_body_size`: Retrieve size of content | body | headers
- `get_content_sie`: Retrieve size of content | body | headers
- `get_content_type` Retrieve or test content type of a HAR request
object
- `get_headers_size` Retrieve size of content | body | headers
- `is_binary`: Retrieve or test content type of a HAR request object
- `is_content_type`: Retrieve or test content type of a HAR request
object
- `is_css`: Retrieve or test content type of a HAR request object
- `is_gif`: Retrieve or test content type of a HAR request object
- `is_html`: Retrieve or test content type of a HAR request object
- `is_javascript`: Retrieve or test content type of a HAR request
object
- `is_jpeg`: Retrieve or test content type of a HAR request object
- `is_json`: Retrieve or test content type of a HAR request object
- `is_plain`: Retrieve or test content type of a HAR request object
- `is_png`: Retrieve or test content type of a HAR request object
- `is_svg`: Retrieve or test content type of a HAR request object
- `is_xhr`: Retrieve or test content type of a HAR request object
- `is_xml`: Retrieve or test content type of a HAR request object
Some functions from `HARtools` are imported/exported and `%>%` is
imported/exported.
### TODO
Suggest more in a feature req!
Suggest more in a feature req\!
- <strike>Implement `render.json`</strike>
- <strike>Implement "file rendering"</strike>
- <strike>Implement `execute` (you can script Splash!)</strike>
- <strike>Add integration with [`HARtools`](https://github.com/johndharrison/HARtools)</strike>
- <strike>*Possibly* writing R function wrappers to install/start/stop Splash</strike> which would also support enabling javascript profiles, request filters and proxy profiles from with R directly, using [`harbor`](https://github.com/wch/harbor)
- Re-implement `render_file()`
- Testing results with all combinations of parameters
- <strike>Implement `render.json`</strike>
- <strike>Implement “file rendering”</strike>
- <strike>Implement `execute` (you can script Splash\!)</strike>
- <strike>Add integration with
[`HARtools`](https://github.com/johndharrison/HARtools)</strike>
- <strike>*Possibly* writing R function wrappers to install/start/stop
Splash</strike> which would also support enabling javascript
profiles, request filters and proxy profiles from with R directly,
using [`harbor`](https://github.com/wch/harbor)
- Re-implement `render_file()`
- Testing results with all combinations of parameters
### Installation
@ -124,7 +195,9 @@ install.packages("splashr", repos = c("https://cinc.rud.is/"))
### Usage
NOTE: ALL of these examples assume Splash is running in the default configuraiton on `localhost` (i.e. started with `start_splash()` or the docker example commands) unless otherwise noted.
NOTE: ALL of these examples assume Splash is running in the default
configuraiton on `localhost` (i.e. started with `start_splash()` or the
docker example commands) unless otherwise noted.
``` r
library(splashr)
@ -137,7 +210,7 @@ library(tidyverse)
packageVersion("splashr")
```
## [1] '0.6.0'
## [1] '0.7.0'
``` r
splash_active()
@ -152,37 +225,39 @@ splash_debug()
## List of 7
## $ active : list()
## $ argcache: int 0
## $ fds : int 19
## $ leaks :List of 4
## ..$ Deferred : int 50
## ..$ LuaRuntime: int 1
## ..$ QTimer : int 1
## ..$ Request : int 1
## $ maxrss : int 191004
## $ fds : int 59
## $ leaks :List of 5
## ..$ Deferred : int 20
## ..$ DelayedCall: int 3
## ..$ LuaRuntime : int 1
## ..$ QTimer : int 1
## ..$ Request : int 1
## $ maxrss : int 219096
## $ qsize : int 0
## $ url : chr "http://localhost:8050"
## - attr(*, "class")= chr [1:2] "splash_debug" "list"
## NULL
Notice the difference between a rendered HTML scrape and a non-rendered one:
Notice the difference between a rendered HTML scrape and a non-rendered
one:
``` r
render_html(url = "http://marvel.com/universe/Captain_America_(Steve_Rogers)")
```
## {xml_document}
## <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr">
## [1] <head>\n<script src="http://widget-cdn.rpxnow.com/manifest/login?version=release%2F1.116.0_widgets_767" type="tex ...
## [2] <body id="index-index" class="index-index" onload="findLinks('myLink');">\n\n\t<div id="page_frame" style="overfl ...
## {html_document}
## <html lang="en">
## [1] <head class="at-element-marker">\n<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">\n<meta char ...
## [2] <body class="terrigen">\n<div id="__next"><div id="terrigen-page" class="page"><div id="page-wrapper" class="page ...
``` r
xml2::read_html("http://marvel.com/universe/Captain_America_(Steve_Rogers)")
```
## {xml_document}
## <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr">
## [1] <head>\n<meta http-equiv="X-UA-Compatible" content="IE=Edge">\n<link href="https://plus.google.com/10852333737344 ...
## [2] <body id="index-index" class="index-index" onload="findLinks('myLink');">\n\n\t<div id="page_frame" style="overfl ...
## {html_document}
## <html lang="en">
## [1] <head>\n<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">\n<meta charset="utf-8" class="next-he ...
## [2] <body class="terrigen">\n<div id="__next"><div id="terrigen-page" class="page"><div id="page-wrapper" class="page ...
You can also profile pages:
@ -196,30 +271,53 @@ print(har)
## HAR specification version: 1.2
## --------HAR CREATOR--------
## Created by: Splash
## version: 3.0
## version: 3.4.1
## --------HAR BROWSER--------
## Browser: QWebKit
## version: 602.1
## --------HAR PAGES--------
## Page id: 1 , Page title: Poynter - A global leader in journalism.
## Page id: 1 , Page title: Poynter - Poynter
## --------HAR ENTRIES--------
## Number of entries: 81
## Number of entries: 270
## REQUESTS:
## Page: 1
## Number of entries: 81
## Number of entries: 270
## - http://www.poynter.org/
## - https://www.poynter.org/
## - https://www.google-analytics.com/analytics.js
## - https://www.googletagservices.com/tag/js/gpt.js
## - https://www.poynter.org/core/themes/stable/css/system/components/ajax-progress.module.css?ovintj
## - https://www.poynter.org/wp-includes/js/wp-emoji-release.min.js?ver=5.3.2
## - https://www.poynter.org/wp-includes/css/dist/block-library/style.min.css?ver=5.3.2
## ........
## - https://www.poynter.org/themes/custom/poynter_theme/fonts/PoynterGothicTextCond/PoynterGothicTextCond-Bold.woff2
## - https://www.poynter.org/themes/custom/poynter_theme/fonts/PoynterOSDisplay/PoynterOSDisplay-Bold.woff2
## - https://www.poynter.org/themes/custom/poynter_theme/fonts/PoynterOSDisplay/PoynterOSDisplay-Semibold.woff2
## - https://www.poynter.org/themes/custom/poynter_theme/fonts/PoynterSerifRE/PoynterSerifRE-Italic.woff2
## - https://www.poynter.org/themes/custom/poynter_theme/fonts/PoynterSerifRE/PoynterSerifRE-BoldItalic.woff2
## - https://www.poynter.org/wp-content/themes/elumine-child/assets/fonts/PoynterOSDisplay/PoynterOSDisplay-BoldItalic...
## - https://securepubads.g.doubleclick.net/gampad/ads?gdfp_req=1&pvsid=3405643238502098&correlator=142496486392385&ou...
## - https://securepubads.g.doubleclick.net/gpt/pubads_impl_rendering_2020021101.js
## - https://tpc.googlesyndication.com/safeframe/1-0-37/html/container.html
## - https://static.chartbeat.com/js/chartbeat.js
``` r
tidy_har(har)
```
You can use [`HARtools::HARviewer`](https://github.com/johndharrison/HARtools/blob/master/R/HARviewer.R) — which this pkg import/exports — to get view the HAR in an interactive HTML widget.
## # A tibble: 270 x 20
## status started total_time page_ref timings req_url req_method req_http_version req_hdr_size req_headers req_cookies
## <int> <chr> <int> <chr> <I<lis> <chr> <chr> <chr> <int> <I<list>> <I<list>>
## 1 301 2020-0… 171 1 <tibbl http:/ GET HTTP/1.1 189 <tibble [2 <tibble [0
## 2 200 2020-0… 238 1 <tibbl https: GET HTTP/1.1 189 <tibble [2 <tibble [0
## 3 200 2020-0… 135 1 <tibbl https: GET HTTP/1.1 164 <tibble [3 <tibble [0
## 4 200 2020-0… 93 1 <tibbl https: GET HTTP/1.1 164 <tibble [3 <tibble [0
## 5 200 2020-0… 158 1 <tibbl https: GET HTTP/1.1 179 <tibble [3 <tibble [0
## 6 200 2020-0… 218 1 <tibbl https: GET HTTP/1.1 179 <tibble [3 <tibble [0
## 7 200 2020-0… 262 1 <tibbl https: GET HTTP/1.1 179 <tibble [3 <tibble [0
## 8 200 2020-0… 265 1 <tibbl https: GET HTTP/1.1 179 <tibble [3 <tibble [0
## 9 200 2020-0… 266 1 <tibbl https: GET HTTP/1.1 179 <tibble [3 <tibble [0
## 10 200 2020-0… 269 1 <tibbl https: GET HTTP/1.1 179 <tibble [3 <tibble [0
## # … with 260 more rows, and 9 more variables: resp_url <chr>, resp_rdrurl <chr>, resp_type <chr>, resp_size <int>,
## # resp_cookies <I<list>>, resp_headers <I<list>>, resp_encoding <chr>, resp_content_size <dbl>, resp_content <chr>
You can use
[`HARtools::HARviewer`](https://github.com/johndharrison/HARtools/blob/master/R/HARviewer.R)
— which this pkg import/exports — to get view the HAR in an interactive
HTML widget.
Full web page snapshots are easy-peasy too:
@ -278,4 +376,6 @@ stop_splash(splash_vm)
### Code of Conduct
Please note that this project is released with a [Contributor Code of Conduct](CONDUCT.md). By participating in this project you agree to abide by its terms.
Please note that this project is released with a [Contributor Code of
Conduct](CONDUCT.md). By participating in this project you agree to
abide by its terms.

BIN
img/cap.jpg

Binary file not shown.

Before

Width:  |  Height:  |  Size: 42 KiB

After

Width:  |  Height:  |  Size: 42 KiB

BIN
img/cap.png

Binary file not shown.

Before

Width:  |  Height:  |  Size: 521 KiB

After

Width:  |  Height:  |  Size: 590 KiB

34
man/as_data_frame.harentry.Rd

@ -1,34 +0,0 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/as-data-frame-har.R
\name{as_data_frame.harentry}
\alias{as_data_frame.harentry}
\alias{as_data_frame.harentries}
\alias{as_data_frame.har}
\alias{as.data.frame.har}
\alias{as.data.frame.harentries}
\alias{as.data.frame.harentry}
\title{Turns a "HAR"-like object into a data frame(tibble)}
\usage{
as_data_frame.harentry(x, ...)
as_data_frame.harentries(x, ...)
as_data_frame.har(x, ...)
\method{as.data.frame}{har}(x, ...)
\method{as.data.frame}{harentries}(x, ...)
\method{as.data.frame}{harentry}(x, ...)
}
\arguments{
\item{x}{A \code{harentry} object}
\item{...}{ignored}
}
\value{
data frame (tibble)
}
\description{
Turns a "HAR"-like object into a data frame(tibble)
}

21
man/execute_lua.Rd

@ -4,8 +4,16 @@
\alias{execute_lua}
\title{Execute a custom rendering script and return a result.}
\usage{
execute_lua(splash_obj, lua_source, timeout = 30, allowed_domains, proxy,
filters, save_args, load_args)
execute_lua(
splash_obj,
lua_source,
timeout = 30,
allowed_domains,
proxy,
filters,
save_args,
load_args
)
}
\arguments{
\item{splash_obj}{Object created by a call to \code{\link[=splash]{splash()}}}
@ -54,8 +62,11 @@ member_scores
}
}
\seealso{
Other splash_renderers: \code{\link{render_har}},
\code{\link{render_html}}, \code{\link{render_jpeg}},
\code{\link{render_json}}, \code{\link{render_png}}
Other splash_renderers:
\code{\link{render_har}()},
\code{\link{render_html}()},
\code{\link{render_jpeg}()},
\code{\link{render_json}()},
\code{\link{render_png}()}
}
\concept{splash_renderers}

18
man/get_content_size.Rd

@ -19,13 +19,15 @@ get_headers_size(har_resp_obj)
Retrieve size of content | body | headers
}
\seealso{
Other splash_har_helpers: \code{\link{get_content_type}},
\code{\link{get_har_entry}},
\code{\link{get_header_val}}, \code{\link{get_headers}},
\code{\link{get_request_type}},
\code{\link{get_request_url}},
\code{\link{get_response_body}},
\code{\link{get_response_url}},
\code{\link{har_entry_count}}
Other splash_har_helpers:
\code{\link{get_content_type}()},
\code{\link{get_har_entry}()},
\code{\link{get_header_val}()},
\code{\link{get_headers}()},
\code{\link{get_request_type}()},
\code{\link{get_request_url}()},
\code{\link{get_response_body}()},
\code{\link{get_response_url}()},
\code{\link{har_entry_count}()}
}
\concept{splash_har_helpers}

18
man/get_content_type.Rd

@ -54,13 +54,15 @@ is_xhr(har_resp_obj)
Retrieve or test content type of a HAR request object
}
\seealso{
Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_har_entry}},
\code{\link{get_header_val}}, \code{\link{get_headers}},
\code{\link{get_request_type}},
\code{\link{get_request_url}},
\code{\link{get_response_body}},
\code{\link{get_response_url}},
\code{\link{har_entry_count}}
Other splash_har_helpers:
\code{\link{get_content_size}()},
\code{\link{get_har_entry}()},
\code{\link{get_header_val}()},
\code{\link{get_headers}()},
\code{\link{get_request_type}()},
\code{\link{get_request_url}()},
\code{\link{get_response_body}()},
\code{\link{get_response_url}()},
\code{\link{har_entry_count}()}
}
\concept{splash_har_helpers}

18
man/get_har_entry.Rd

@ -15,13 +15,15 @@ get_har_entry(x, i = 1)
Retrieve an entry by index from a HAR object
}
\seealso{
Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_content_type}},
\code{\link{get_header_val}}, \code{\link{get_headers}},
\code{\link{get_request_type}},
\code{\link{get_request_url}},
\code{\link{get_response_body}},
\code{\link{get_response_url}},
\code{\link{har_entry_count}}
Other splash_har_helpers:
\code{\link{get_content_size}()},
\code{\link{get_content_type}()},
\code{\link{get_header_val}()},
\code{\link{get_headers}()},
\code{\link{get_request_type}()},
\code{\link{get_request_url}()},
\code{\link{get_response_body}()},
\code{\link{get_response_url}()},
\code{\link{har_entry_count}()}
}
\concept{splash_har_helpers}

18
man/get_header_val.Rd

@ -20,13 +20,15 @@ as is the passed-in requested header. Also, if there is more than one only
the first is returned.
}
\seealso{
Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_content_type}},
\code{\link{get_har_entry}}, \code{\link{get_headers}},
\code{\link{get_request_type}},
\code{\link{get_request_url}},
\code{\link{get_response_body}},
\code{\link{get_response_url}},
\code{\link{har_entry_count}}
Other splash_har_helpers:
\code{\link{get_content_size}()},
\code{\link{get_content_type}()},
\code{\link{get_har_entry}()},
\code{\link{get_headers}()},
\code{\link{get_request_type}()},
\code{\link{get_request_url}()},
\code{\link{get_response_body}()},
\code{\link{get_response_url}()},
\code{\link{har_entry_count}()}
}
\concept{splash_har_helpers}

19
man/get_headers.Rd

@ -16,14 +16,15 @@ Retrieve response headers as a data frame
the \code{name} column that contains the header key is normalized to lower case
}
\seealso{
Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_content_type}},
\code{\link{get_har_entry}},
\code{\link{get_header_val}},
\code{\link{get_request_type}},
\code{\link{get_request_url}},
\code{\link{get_response_body}},
\code{\link{get_response_url}},
\code{\link{har_entry_count}}
Other splash_har_helpers:
\code{\link{get_content_size}()},
\code{\link{get_content_type}()},
\code{\link{get_har_entry}()},
\code{\link{get_header_val}()},
\code{\link{get_request_type}()},
\code{\link{get_request_url}()},
\code{\link{get_response_body}()},
\code{\link{get_response_url}()},
\code{\link{har_entry_count}()}
}
\concept{splash_har_helpers}

18
man/get_request_type.Rd

@ -19,13 +19,15 @@ is_post(har_resp_obj)
Retrieve or test request type
}
\seealso{
Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_content_type}},
\code{\link{get_har_entry}},
\code{\link{get_header_val}}, \code{\link{get_headers}},
\code{\link{get_request_url}},
\code{\link{get_response_body}},
\code{\link{get_response_url}},
\code{\link{har_entry_count}}
Other splash_har_helpers:
\code{\link{get_content_size}()},
\code{\link{get_content_type}()},
\code{\link{get_har_entry}()},
\code{\link{get_header_val}()},
\code{\link{get_headers}()},
\code{\link{get_request_url}()},
\code{\link{get_response_body}()},
\code{\link{get_response_url}()},
\code{\link{har_entry_count}()}
}
\concept{splash_har_helpers}

18
man/get_request_url.Rd

@ -13,13 +13,15 @@ get_request_url(har_resp_obj)
Retrieve request URL
}
\seealso{
Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_content_type}},
\code{\link{get_har_entry}},
\code{\link{get_header_val}}, \code{\link{get_headers}},
\code{\link{get_request_type}},
\code{\link{get_response_body}},
\code{\link{get_response_url}},
\code{\link{har_entry_count}}
Other splash_har_helpers:
\code{\link{get_content_size}()},
\code{\link{get_content_type}()},
\code{\link{get_har_entry}()},
\code{\link{get_header_val}()},
\code{\link{get_headers}()},
\code{\link{get_request_type}()},
\code{\link{get_response_body}()},
\code{\link{get_response_url}()},
\code{\link{har_entry_count}()}
}
\concept{splash_har_helpers}

18
man/get_response_body.Rd

@ -19,13 +19,15 @@ A \code{raw} vector of the content or \code{NULL} or a \code{character} if \code
Retrieve the body content of a HAR entry
}
\seealso{
Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_content_type}},
\code{\link{get_har_entry}},
\code{\link{get_header_val}}, \code{\link{get_headers}},
\code{\link{get_request_type}},
\code{\link{get_request_url}},
\code{\link{get_response_url}},
\code{\link{har_entry_count}}
Other splash_har_helpers:
\code{\link{get_content_size}()},
\code{\link{get_content_type}()},
\code{\link{get_har_entry}()},
\code{\link{get_header_val}()},
\code{\link{get_headers}()},
\code{\link{get_request_type}()},
\code{\link{get_request_url}()},
\code{\link{get_response_url}()},
\code{\link{har_entry_count}()}
}
\concept{splash_har_helpers}

18
man/get_response_url.Rd

@ -13,13 +13,15 @@ get_response_url(har_resp_obj)
Retrieve response URL
}
\seealso{
Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_content_type}},
\code{\link{get_har_entry}},
\code{\link{get_header_val}}, \code{\link{get_headers}},
\code{\link{get_request_type}},
\code{\link{get_request_url}},
\code{\link{get_response_body}},
\code{\link{har_entry_count}}
Other splash_har_helpers:
\code{\link{get_content_size}()},
\code{\link{get_content_type}()},
\code{\link{get_har_entry}()},
\code{\link{get_header_val}()},
\code{\link{get_headers}()},
\code{\link{get_request_type}()},
\code{\link{get_request_url}()},
\code{\link{get_response_body}()},
\code{\link{har_entry_count}()}
}
\concept{splash_har_helpers}

18
man/har_entry_count.Rd

@ -13,13 +13,15 @@ har_entry_count(x)
Retrieves number of HAR entries in a response
}
\seealso{
Other splash_har_helpers: \code{\link{get_content_size}},
\code{\link{get_content_type}},
\code{\link{get_har_entry}},
\code{\link{get_header_val}}, \code{\link{get_headers}},
\code{\link{get_request_type}},
\code{\link{get_request_url}},
\code{\link{get_response_body}},
\code{\link{get_response_url}}
Other splash_har_helpers:
\code{\link{get_content_size}()},
\code{\link{get_content_type}()},
\code{\link{get_har_entry}()},
\code{\link{get_header_val}()},
\code{\link{get_headers}()},
\code{\link{get_request_type}()},
\code{\link{get_request_url}()},
\code{\link{get_response_body}()},
\code{\link{get_response_url}()}
}
\concept{splash_har_helpers}

5
man/install_splash.Rd

@ -23,7 +23,8 @@ stop_splash(splash_container)
}
}
\seealso{
Other splash_docker_helpers: \code{\link{start_splash}},
\code{\link{stop_splash}}
Other splash_docker_helpers:
\code{\link{start_splash}()},
\code{\link{stop_splash}()}
}
\concept{splash_docker_helpers}

50
man/render_har.Rd

@ -4,23 +4,43 @@
\alias{render_har}
\title{Return information about Splash interaction with a website in HAR format.}
\usage{
render_har(splash_obj = splash_local, url, base_url,
response_body = FALSE, timeout = 30, resource_timeout, wait = 0,
proxy, js, js_src, filters, allowed_domains, allowed_content_types,
forbidden_content_types, viewport = "1024x768", images, headers, body,
http_method, save_args, load_args)
render_har(
splash_obj = splash_local,
url,
base_url,
response_body = FALSE,
timeout = 30,
resource_timeout,
wait = 0,
proxy,
js,
js_src,
filters,
allowed_domains,
allowed_content_types,
forbidden_content_types,
viewport = "1024x768",
images,
headers,
body,
http_method,
save_args,
load_args,
http2 = FALSE,
engine = c("webkit", "chromium")
)
}
\arguments{
\item{splash_obj}{Object created by a call to \code{\link[=splash]{splash()}}}
\item{url}{The URL to render (required)}
\item{base_url}{The base url to render the page with.}
\item{base_url}{The base URL to render the page with.}
\item{response_body}{When \code{TRUE}, response content is included in the HAR records}
\item{timeout}{A timeout (in seconds) for the render (defaults to 30). Without
reconfiguring the startup parameters of the Splash server (not this package)
re-configuring the start-up parameters of the Splash server (not this package)
the maximum allowed value for the timeout is 60 seconds.}
\item{resource_timeout}{A timeout (in seconds) for individual network requests.}
@ -48,7 +68,7 @@ present, Splash will abort any request if the response’s content type matches
any of the content types in this list. Wildcards are supported.}
\item{viewport}{View width and height (in pixels) of the browser viewport to render the
web page. Format is “<width>x<height>”, e.g. 800x600. Default value is "full".}
web page. Format is “width>xheight”, e.g. 800x600. Default value is "full".}
\item{images}{Whether to download images.}
@ -61,6 +81,11 @@ web page. Format is “<width>x<height>”, e.g. 800x600. Default value is "full
\item{save_args}{A list of argument names to put in cache.}
\item{load_args}{Parameter values to load from cache}
\item{http2}{Enable or disable HTTP2 support. \code{TRUE} to enable; \code{FALSE} to disable; defaults to \code{FALSE}
when \code{engine} is \code{webkit} due to malformed behaviour in 3.4.x of Splash}
\item{engine}{one of \code{webkit} or \code{chromium}; defaults to \code{webkit}}
}
\value{
a \link{HARtools} \code{har} object
@ -73,8 +98,11 @@ is incredibly detailed, full of information on every component loaded.
\href{http://splash.readthedocs.io/en/stable/index.html}{Splash docs}
}
\seealso{
Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_html}}, \code{\link{render_jpeg}},
\code{\link{render_json}}, \code{\link{render_png}}
Other splash_renderers:
\code{\link{execute_lua}()},
\code{\link{render_html}()},
\code{\link{render_jpeg}()},
\code{\link{render_json}()},
\code{\link{render_png}()}
}
\concept{splash_renderers}

50
man/render_html.Rd

@ -4,21 +4,41 @@
\alias{render_html}
\title{Return the HTML of the javascript-rendered page.}
\usage{
render_html(splash_obj = splash_local, url, base_url, timeout = 30,
resource_timeout, wait = 0, proxy, js, js_src, filters,
allowed_domains, allowed_content_types, forbidden_content_types,
viewport = "1024x768", images, headers, body, http_method, save_args,
load_args, raw_html = FALSE)
render_html(
splash_obj = splash_local,
url,
base_url,
timeout = 30,
resource_timeout,
wait = 0,
proxy,
js,
js_src,
filters,
allowed_domains,
allowed_content_types,
forbidden_content_types,
viewport = "1024x768",
images,
headers,
body,
http_method,
save_args,
load_args,
http2 = FALSE,
engine = c("webkit", "chromium"),
raw_html = FALSE
)
}
\arguments{
\item{splash_obj}{Object created by a call to \code{\link[=splash]{splash()}}}
\item{url}{The URL to render (required)}
\item{base_url}{The base url to render the page with.}
\item{base_url}{The base URL to render the page with.}
\item{timeout}{A timeout (in seconds) for the render (defaults to 30). Without
reconfiguring the startup parameters of the Splash server (not this package)
re-configuring the start-up parameters of the Splash server (not this package)
the maximum allowed value for the timeout is 60 seconds.}
\item{resource_timeout}{A timeout (in seconds) for individual network requests.}
@ -46,7 +66,7 @@ present, Splash will abort any request if the response’s content type matches
any of the content types in this list. Wildcards are supported.}
\item{viewport}{View width and height (in pixels) of the browser viewport to render the
web page. Format is “<width>x<height>”, e.g. 800x600. Default value is "full".}
web page. Format is “width>xheight”, e.g. 800x600. Default value is "full".}
\item{images}{Whether to download images.}
@ -60,6 +80,11 @@ web page. Format is “<width>x<height>”, e.g. 800x600. Default value is "full
\item{load_args}{Parameter values to load from cache}
\item{http2}{Enable or disable HTTP2 support. \code{TRUE} to enable; \code{FALSE} to disable; defaults to \code{FALSE}
when \code{engine} is \code{webkit} due to malformed behaviour in 3.4.x of Splash}
\item{engine}{one of \code{webkit} or \code{chromium}; defaults to \code{webkit}}
\item{raw_html}{if \code{TRUE} then return a character vector vs an XML document. Only valid for \code{render_html}}
}
\value{
@ -75,8 +100,11 @@ Similar (i.e. a dynamic equivalent) to \code{rvest::read_html}.
\href{http://splash.readthedocs.io/en/stable/index.html}{Splash docs}
}
\seealso{
Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_jpeg}},
\code{\link{render_json}}, \code{\link{render_png}}
Other splash_renderers:
\code{\link{execute_lua}()},
\code{\link{render_har}()},
\code{\link{render_jpeg}()},
\code{\link{render_json}()},
\code{\link{render_png}()}
}
\concept{splash_renderers}

54
man/render_jpeg.Rd

@ -4,19 +4,41 @@
\alias{render_jpeg}
\title{Return a image (in JPEG format) of the javascript-rendered page.}
\usage{
render_jpeg(splash_obj = splash_local, url, base_url = NULL,
quality = 75, width, height, timeout = 30, resource_timeout,
wait = 0, render_all = TRUE, proxy, js, js_src, filters,
allowed_domains, allowed_content_types, forbidden_content_types,
viewport = "full", images, headers, body, http_method, save_args,
load_args)
render_jpeg(
splash_obj = splash_local,
url,
base_url = NULL,
quality = 75,
width,
height,
timeout = 30,
resource_timeout,
wait = 0,
render_all = TRUE,
proxy,
js,
js_src,
filters,
allowed_domains,
allowed_content_types,
forbidden_content_types,
viewport = "full",
images,
headers,
body,
http_method,
save_args,
load_args,
http2 = FALSE,
engine = c("webkit", "chromium")
)
}
\arguments{
\item{splash_obj}{Object created by a call to \code{\link[=splash]{splash()}}}
\item{url}{The URL to render (required)}
\item{base_url}{The base url to render the page with.}
\item{base_url}{The base URL to render the page with.}
\item{quality}{JPEG quality parameter in range from 0 to 100. Default is quality=75.}
@ -27,7 +49,7 @@ pixels) keeping the aspect ratio. These are optional}
pixels) keeping the aspect ratio. These are optional}
\item{timeout}{A timeout (in seconds) for the render (defaults to 30). Without
reconfiguring the startup parameters of the Splash server (not this package)
re-configuring the start-up parameters of the Splash server (not this package)
the maximum allowed value for the timeout is 60 seconds.}
\item{resource_timeout}{A timeout (in seconds) for individual network requests.}
@ -58,7 +80,7 @@ present, Splash will abort any request if the response’s content type matches
any of the content types in this list. Wildcards are supported.}
\item{viewport}{View width and height (in pixels) of the browser viewport to render the
web page. Format is “<width>x<height>”, e.g. 800x600. Default value is "full".}
web page. Format is “width>xheight”, e.g. 800x600. Default value is "full".}
\item{images}{Whether to download images.}
@ -71,6 +93,11 @@ web page. Format is “<width>x<height>”, e.g. 800x600. Default value is "full
\item{save_args}{A list of argument names to put in cache.}
\item{load_args}{Parameter values to load from cache}
\item{http2}{Enable or disable HTTP2 support. \code{TRUE} to enable; \code{FALSE} to disable; defaults to \code{FALSE}
when \code{engine} is \code{webkit} due to malformed behaviour in 3.4.x of Splash}
\item{engine}{one of \code{webkit} or \code{chromium}; defaults to \code{webkit}}
}
\value{
a \link{magick} image object
@ -82,8 +109,11 @@ Return a image (in JPEG format) of the javascript-rendered page.
\href{http://splash.readthedocs.io/en/stable/index.html}{Splash docs}
}
\seealso{
Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_html}},
\code{\link{render_json}}, \code{\link{render_png}}
Other splash_renderers:
\code{\link{execute_lua}()},
\code{\link{render_har}()},
\code{\link{render_html}()},
\code{\link{render_json}()},
\code{\link{render_png}()}
}
\concept{splash_renderers}

58
man/render_json.Rd

@ -4,21 +4,48 @@
\alias{render_json}
\title{Return a json-encoded dictionary with information about javascript-rendered webpage.}
\usage{
render_json(splash_obj = splash_local, url, base_url = NULL,
quality = 75, width, height, timeout = 30, resource_timeout,
wait = 0, render_all = FALSE, proxy, js, js_src, filters,
allowed_domains, allowed_content_types, forbidden_content_types,
viewport = "1024x768", images, headers, body, http_method, save_args,
load_args, html = TRUE, png = FALSE, jpeg = FALSE,
iframes = TRUE, script = TRUE, console = TRUE, history = TRUE,
har = TRUE, response_body = FALSE)
render_json(
splash_obj = splash_local,
url,
base_url = NULL,
quality = 75,
width,
height,
timeout = 30,
resource_timeout,
wait = 0,
render_all = FALSE,
proxy,
js,
js_src,
filters,
allowed_domains,
allowed_content_types,
forbidden_content_types,
viewport = "1024x768",
images,
headers,
body,
http_method,
save_args,
load_args,
html = TRUE,
png = FALSE,
jpeg = FALSE,
iframes = TRUE,
script = TRUE,
console = TRUE,
history = TRUE,
har = TRUE,
response_body = FALSE
)
}
\arguments{
\item{splash_obj}{Object created by a call to \code{\link[=splash]{splash()}}}
\item{url}{The URL to render (required)}
\item{base_url}{The base url to render the page with.}
\item{base_url}{The base URL to render the page with.}
\item{quality}{JPEG quality parameter in range from 0 to 100. Default is quality=75.}
@ -29,7 +56,7 @@ pixels) keeping the aspect ratio. These are optional}
pixels) keeping the aspect ratio. These are optional}
\item{timeout}{A timeout (in seconds) for the render (defaults to 30). Without
reconfiguring the startup parameters of the Splash server (not this package)
re-configuring the start-up parameters of the Splash server (not this package)
the maximum allowed value for the timeout is 60 seconds.}
\item{resource_timeout}{A timeout (in seconds) for individual network requests.}
@ -60,7 +87,7 @@ present, Splash will abort any request if the response’s content type matches
any of the content types in this list. Wildcards are supported.}
\item{viewport}{View width and height (in pixels) of the browser viewport to render the
web page. Format is “<width>x<height>”, e.g. 800x600. Default value is "full".}
web page. Format is “width>xheight”, e.g. 800x600. Default value is "full".}
\item{images}{Whether to download images.}
@ -114,8 +141,11 @@ overwhelmed with data. Use \code{\link[=str]{str()}} to inspect various portions
\href{http://splash.readthedocs.io/en/stable/index.html}{Splash docs}
}
\seealso{
Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_html}},
\code{\link{render_jpeg}}, \code{\link{render_png}}
Other splash_renderers:
\code{\link{execute_lua}()},
\code{\link{render_har}()},
\code{\link{render_html}()},
\code{\link{render_jpeg}()},
\code{\link{render_png}()}
}
\concept{splash_renderers}

52
man/render_png.Rd

@ -4,24 +4,46 @@
\alias{render_png}
\title{Return an image (in PNG format) of the javascript-rendered page.}
\usage{
render_png(splash_obj = splash_local, url, base_url = NULL, width,
height, timeout = 30, resource_timeout, wait = 0,
render_all = TRUE, proxy, js, js_src, filters, allowed_domains,
allowed_content_types, forbidden_content_types, viewport = "full",
images, headers, body, http_method, save_args, load_args)
render_png(
splash_obj = splash_local,
url,
base_url = NULL,
width,
height,
timeout = 30,
resource_timeout,
wait = 0,
render_all = TRUE,
proxy,
js,
js_src,
filters,
allowed_domains,
allowed_content_types,
forbidden_content_types,
viewport = "full",
images,
headers,
body,
http_method,
save_args,
load_args,
http2 = FALSE,
engine = c("webkit", "chromium")
)
}
\arguments{
\item{splash_obj}{Object created by a call to \code{\link[=splash]{splash()}}}
\item{url}{The URL to render (required)}
\item{base_url}{The base url to render the page with.}
\item{base_url}{The base URL to render the page with.}
\item{width, height}{Resize the rendered image to the given width/height (in
pixels) keeping the aspect ratio. These are optional}
\item{timeout}{A timeout (in seconds) for the render (defaults to 30). Without
reconfiguring the startup parameters of the Splash server (not this package)
re-configuring the start-up parameters of the Splash server (not this package)
the maximum allowed value for the timeout is 60 seconds.}
\item{resource_timeout}{A timeout (in seconds) for individual network requests.}
@ -52,7 +74,7 @@ present, Splash will abort any request if the response’s content type matches
any of the content types in this list. Wildcards are supported.}
\item{viewport}{View width and height (in pixels) of the browser viewport to render the
web page. Format is “<width>x<height>”, e.g. 800x600. Default value is "full".}
web page. Format is “width>xheight”, e.g. 800x600. Default value is "full".}
\item{images}{Whether to download images.}
@ -65,6 +87,11 @@ web page. Format is “<width>x<height>”, e.g. 800x600. Default value is "full
\item{save_args}{A list of argument names to put in cache.}
\item{load_args}{Parameter values to load from cache}
\item{http2}{Enable or disable HTTP2 support. \code{TRUE} to enable; \code{FALSE} to disable; defaults to \code{FALSE}
when \code{engine} is \code{webkit} due to malformed behaviour in 3.4.x of Splash}
\item{engine}{one of \code{webkit} or \code{chromium}; defaults to \code{webkit}}
}
\value{
a \link{magick} image object
@ -81,8 +108,11 @@ render_png(url = "https://httpbin.org/")
\href{http://splash.readthedocs.io/en/stable/index.html}{Splash docs}
}
\seealso{
Other splash_renderers: \code{\link{execute_lua}},
\code{\link{render_har}}, \code{\link{render_html}},
\code{\link{render_jpeg}}, \code{\link{render_json}}
Other splash_renderers:
\code{\link{execute_lua}()},
\code{\link{render_har}()},
\code{\link{render_html}()},
\code{\link{render_jpeg}()},
\code{\link{render_json}()}
}
\concept{splash_renderers}

9
man/splash_active.Rd

@ -22,9 +22,10 @@ splash_active(sp)
}
}
\seealso{
Other splash_info_functions: \code{\link{splash_debug}},
\code{\link{splash_history}},
\code{\link{splash_perf_stats}},
\code{\link{splash_version}}
Other splash_info_functions:
\code{\link{splash_debug}()},
\code{\link{splash_history}()},
\code{\link{splash_perf_stats}()},
\code{\link{splash_version}()}
}
\concept{splash_info_functions}

20
man/splash_add_lua.Rd

@ -27,12 +27,18 @@ end
If you need more flexibility, use the \code{\link[=execute_lua]{execute_lua()}} function.
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_click}},
\code{\link{splash_focus}}, \code{\link{splash_go}},
\code{\link{splash_har_reset}}, \code{\link{splash_har}},
\code{\link{splash_html}}, \code{\link{splash_png}},
\code{\link{splash_press}}, \code{\link{splash_release}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_click}()},
\code{\link{splash_focus}()},
\code{\link{splash_go}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_har}()},
\code{\link{splash_html}()},
\code{\link{splash_png}()},
\code{\link{splash_press}()},
\code{\link{splash_release}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_send_text}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

20
man/splash_click.Rd

@ -15,12 +15,18 @@ splash_click(splash_obj, x, y)
Trigger mouse click event in web page.
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_focus}}, \code{\link{splash_go}},
\code{\link{splash_har_reset}}, \code{\link{splash_har}},
\code{\link{splash_html}}, \code{\link{splash_png}},
\code{\link{splash_press}}, \code{\link{splash_release}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_focus}()},
\code{\link{splash_go}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_har}()},
\code{\link{splash_html}()},
\code{\link{splash_png}()},
\code{\link{splash_press}()},
\code{\link{splash_release}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_send_text}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

9
man/splash_debug.Rd

@ -22,10 +22,11 @@ splash_debug(sp)
}
}
\seealso{
Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_history}},
\code{\link{splash_perf_stats}},
\code{\link{splash_version}}
Other splash_info_functions:
\code{\link{splash_active}()},
\code{\link{splash_history}()},
\code{\link{splash_perf_stats}()},
\code{\link{splash_version}()}
}
\concept{splash_info_functions}
\keyword{internal}

9
man/splash_enable_javascript.Rd

@ -27,9 +27,10 @@ splash_local \%>\%
}
}
\seealso{
Other splash_dsl_attributes: \code{\link{splash_images}},
\code{\link{splash_plugins}},
\code{\link{splash_private_mode}},
\code{\link{splash_response_body}}
Other splash_dsl_attributes:
\code{\link{splash_images}()},
\code{\link{splash_plugins}()},
\code{\link{splash_private_mode}()},
\code{\link{splash_response_body}()}
}
\concept{splash_dsl_attributes}

20
man/splash_focus.Rd

@ -18,12 +18,18 @@ Focus on a document element provided by a CSS selector
See \href{https://splash.readthedocs.io/en/stable/scripting-ref.html#splash-send-text}{the docs} for more info
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_click}}, \code{\link{splash_go}},
\code{\link{splash_har_reset}}, \code{\link{splash_har}},
\code{\link{splash_html}}, \code{\link{splash_png}},
\code{\link{splash_press}}, \code{\link{splash_release}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_click}()},
\code{\link{splash_go}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_har}()},
\code{\link{splash_html}()},
\code{\link{splash_png}()},
\code{\link{splash_press}()},
\code{\link{splash_release}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_send_text}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

20
man/splash_go.Rd

@ -26,12 +26,18 @@ splash_local \%>\%
}
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_click}}, \code{\link{splash_focus}},
\code{\link{splash_har_reset}}, \code{\link{splash_har}},
\code{\link{splash_html}}, \code{\link{splash_png}},
\code{\link{splash_press}}, \code{\link{splash_release}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_click}()},
\code{\link{splash_focus}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_har}()},
\code{\link{splash_html}()},
\code{\link{splash_png}()},
\code{\link{splash_press}()},
\code{\link{splash_release}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_send_text}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

20
man/splash_har.Rd

@ -24,12 +24,18 @@ splash_local \%>\%
}
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_click}}, \code{\link{splash_focus}},
\code{\link{splash_go}}, \code{\link{splash_har_reset}},
\code{\link{splash_html}}, \code{\link{splash_png}},
\code{\link{splash_press}}, \code{\link{splash_release}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_click}()},
\code{\link{splash_focus}()},
\code{\link{splash_go}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_html}()},
\code{\link{splash_png}()},
\code{\link{splash_press}()},
\code{\link{splash_release}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_send_text}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

20
man/splash_har_reset.Rd

@ -13,12 +13,18 @@ splash_har_reset(splash_obj)
Drops all internally stored HAR records.
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_click}}, \code{\link{splash_focus}},
\code{\link{splash_go}}, \code{\link{splash_har}},
\code{\link{splash_html}}, \code{\link{splash_png}},
\code{\link{splash_press}}, \code{\link{splash_release}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_click}()},
\code{\link{splash_focus}()},
\code{\link{splash_go}()},
\code{\link{splash_har}()},
\code{\link{splash_html}()},
\code{\link{splash_png}()},
\code{\link{splash_press}()},
\code{\link{splash_release}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_send_text}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

9
man/splash_history.Rd

@ -19,9 +19,10 @@ splash_history(sp)
}
}
\seealso{
Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_debug}},
\code{\link{splash_perf_stats}},
\code{\link{splash_version}}
Other splash_info_functions:
\code{\link{splash_active}()},
\code{\link{splash_debug}()},
\code{\link{splash_perf_stats}()},
\code{\link{splash_version}()}
}
\concept{splash_info_functions}

20
man/splash_html.Rd

@ -26,12 +26,18 @@ splash_local \%>\%
}
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_click}}, \code{\link{splash_focus}},
\code{\link{splash_go}}, \code{\link{splash_har_reset}},
\code{\link{splash_har}}, \code{\link{splash_png}},
\code{\link{splash_press}}, \code{\link{splash_release}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_click}()},
\code{\link{splash_focus}()},
\code{\link{splash_go}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_har}()},
\code{\link{splash_png}()},
\code{\link{splash_press}()},
\code{\link{splash_release}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_send_text}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

9
man/splash_images.Rd

@ -28,9 +28,10 @@ splash_local \%>\%
}
}
\seealso{
Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_plugins}},
\code{\link{splash_private_mode}},
\code{\link{splash_response_body}}
Other splash_dsl_attributes:
\code{\link{splash_enable_javascript}()},
\code{\link{splash_plugins}()},
\code{\link{splash_private_mode}()},
\code{\link{splash_response_body}()}
}
\concept{splash_dsl_attributes}

8
man/splash_perf_stats.Rd

@ -19,8 +19,10 @@ splash_perf_stats(sp)
}
}
\seealso{
Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_debug}}, \code{\link{splash_history}},
\code{\link{splash_version}}
Other splash_info_functions:
\code{\link{splash_active}()},
\code{\link{splash_debug}()},
\code{\link{splash_history}()},
\code{\link{splash_version}()}
}
\concept{splash_info_functions}

9
man/splash_plugins.Rd

@ -25,9 +25,10 @@ splash_local \%>\%
}
}
\seealso{
Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_images}},
\code{\link{splash_private_mode}},
\code{\link{splash_response_body}}
Other splash_dsl_attributes:
\code{\link{splash_enable_javascript}()},
\code{\link{splash_images}()},
\code{\link{splash_private_mode}()},
\code{\link{splash_response_body}()}
}
\concept{splash_dsl_attributes}

20
man/splash_png.Rd

@ -26,12 +26,18 @@ splash_local \%>\%
}
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_click}}, \code{\link{splash_focus}},
\code{\link{splash_go}}, \code{\link{splash_har_reset}},
\code{\link{splash_har}}, \code{\link{splash_html}},
\code{\link{splash_press}}, \code{\link{splash_release}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_click}()},
\code{\link{splash_focus}()},
\code{\link{splash_go}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_har}()},
\code{\link{splash_html}()},
\code{\link{splash_press}()},
\code{\link{splash_release}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_send_text}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

20
man/splash_press.Rd

@ -15,12 +15,18 @@ splash_press(splash_obj, x, y)
Trigger mouse press event in web page.
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_click}}, \code{\link{splash_focus}},
\code{\link{splash_go}}, \code{\link{splash_har_reset}},
\code{\link{splash_har}}, \code{\link{splash_html}},
\code{\link{splash_png}}, \code{\link{splash_release}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_click}()},
\code{\link{splash_focus}()},
\code{\link{splash_go}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_har}()},
\code{\link{splash_html}()},
\code{\link{splash_png}()},
\code{\link{splash_release}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_send_text}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

9
man/splash_private_mode.Rd

@ -28,9 +28,10 @@ splash_local \%>\%
}
}
\seealso{
Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_images}},
\code{\link{splash_plugins}},
\code{\link{splash_response_body}}
Other splash_dsl_attributes:
\code{\link{splash_enable_javascript}()},
\code{\link{splash_images}()},
\code{\link{splash_plugins}()},
\code{\link{splash_response_body}()}
}
\concept{splash_dsl_attributes}

20
man/splash_release.Rd

@ -15,12 +15,18 @@ splash_release(splash_obj, x, y)
Trigger mouse release event in web page.
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_click}}, \code{\link{splash_focus}},
\code{\link{splash_go}}, \code{\link{splash_har_reset}},
\code{\link{splash_har}}, \code{\link{splash_html}},
\code{\link{splash_png}}, \code{\link{splash_press}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_click}()},
\code{\link{splash_focus}()},
\code{\link{splash_go}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_har}()},
\code{\link{splash_html}()},
\code{\link{splash_png}()},
\code{\link{splash_press}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_send_text}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

9
man/splash_response_body.Rd

@ -25,9 +25,10 @@ splash_local \%>\%
}
}
\seealso{
Other splash_dsl_attributes: \code{\link{splash_enable_javascript}},
\code{\link{splash_images}},
\code{\link{splash_plugins}},
\code{\link{splash_private_mode}}
Other splash_dsl_attributes:
\code{\link{splash_enable_javascript}()},
\code{\link{splash_images}()},
\code{\link{splash_plugins}()},
\code{\link{splash_private_mode}()}
}
\concept{splash_dsl_attributes}

20
man/splash_send_keys.Rd

@ -24,12 +24,18 @@ This is different from \code{\link[=splash_send_text]{splash_send_text()}}
See \href{https://splash.readthedocs.io/en/stable/scripting-ref.html#splash-send-keys}{the docs} for more info
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_click}}, \code{\link{splash_focus}},
\code{\link{splash_go}}, \code{\link{splash_har_reset}},
\code{\link{splash_har}}, \code{\link{splash_html}},
\code{\link{splash_png}}, \code{\link{splash_press}},
\code{\link{splash_release}},
\code{\link{splash_send_text}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_click}()},
\code{\link{splash_focus}()},
\code{\link{splash_go}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_har}()},
\code{\link{splash_html}()},
\code{\link{splash_png}()},
\code{\link{splash_press}()},
\code{\link{splash_release}()},
\code{\link{splash_send_text}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

20
man/splash_send_text.Rd

@ -21,12 +21,18 @@ This adds a call to \code{splash:wait} so you do not have to
See \href{https://splash.readthedocs.io/en/stable/scripting-ref.html#splash-send-keys}{the docs} for more info
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_click}}, \code{\link{splash_focus}},
\code{\link{splash_go}}, \code{\link{splash_har_reset}},
\code{\link{splash_har}}, \code{\link{splash_html}},
\code{\link{splash_png}}, \code{\link{splash_press}},
\code{\link{splash_release}},
\code{\link{splash_send_keys}}, \code{\link{splash_wait}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_click}()},
\code{\link{splash_focus}()},
\code{\link{splash_go}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_har}()},
\code{\link{splash_html}()},
\code{\link{splash_png}()},
\code{\link{splash_press}()},
\code{\link{splash_release}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_wait}()}
}
\concept{splash_dsl_functions}

2
man/splash_user_agent.Rd

@ -62,7 +62,7 @@ ua_chromecast
\arguments{
\item{splash_obj}{splashr object}
\item{user_agent}{1 element character vector, defaults to \code{splashr/#.#.#}.}
\item{user_agent}{1 element character vector, defaults to \verb{splashr/#.#.#}.}
}
\description{
There are a few built-in user agents, all beginning with \code{ua_}.

8
man/splash_version.Rd

@ -19,8 +19,10 @@ splash_version(sp)
}
}
\seealso{
Other splash_info_functions: \code{\link{splash_active}},
\code{\link{splash_debug}}, \code{\link{splash_history}},
\code{\link{splash_perf_stats}}
Other splash_info_functions:
\code{\link{splash_active}()},
\code{\link{splash_debug}()},
\code{\link{splash_history}()},
\code{\link{splash_perf_stats}()}
}
\concept{splash_info_functions}

21
man/splash_wait.Rd

@ -25,13 +25,18 @@ splash_local \%>\%
}
}
\seealso{
Other splash_dsl_functions: \code{\link{splash_add_lua}},
\code{\link{splash_click}}, \code{\link{splash_focus}},
\code{\link{splash_go}}, \code{\link{splash_har_reset}},
\code{\link{splash_har}}, \code{\link{splash_html}},
\code{\link{splash_png}}, \code{\link{splash_press}},
\code{\link{splash_release}},
\code{\link{splash_send_keys}},
\code{\link{splash_send_text}}
Other splash_dsl_functions:
\code{\link{splash_add_lua}()},
\code{\link{splash_click}()},
\code{\link{splash_focus}()},
\code{\link{splash_go}()},
\code{\link{splash_har_reset}()},
\code{\link{splash_har}()},
\code{\link{splash_html}()},
\code{\link{splash_png}()},
\code{\link{splash_press}()},
\code{\link{splash_release}()},
\code{\link{splash_send_keys}()},
\code{\link{splash_send_text}()}
}
\concept{splash_dsl_functions}

1
man/splashr.Rd

@ -3,7 +3,6 @@
\docType{package}
\name{splashr}
\alias{splashr}
\alias{splashr-package}
\title{Tools to Work with the 'Splash' JavaScript Rendering Service}
\description{
'Splash' \url{https://github.com/scrapinghub/splash} is a 'JavaScript' rendering service.

8
man/start_splash.Rd

@ -4,8 +4,7 @@
\alias{start_splash}
\title{Start a Splash server Docker container}
\usage{
start_splash(tag = "latest", container_name = "splashr",
remove = FALSE, ...)
start_splash(tag = "latest", container_name = "splashr", remove = FALSE, ...)
}
\arguments{
\item{tag}{Splash Docker image tag to start}
@ -43,7 +42,8 @@ stop_splash(splash_container)
}
}
\seealso{
Other splash_docker_helpers: \code{\link{install_splash}},
\code{\link{stop_splash}}
Other splash_docker_helpers:
\code{\link{install_splash}()},
\code{\link{stop_splash}()}
}
\concept{splash_docker_helpers}

5
man/stop_splash.Rd

@ -25,7 +25,8 @@ stop_splash(splash_container)
}
}
\seealso{
Other splash_docker_helpers: \code{\link{install_splash}},
\code{\link{start_splash}}
Other splash_docker_helpers:
\code{\link{install_splash}()},
\code{\link{start_splash}()}
}
\concept{splash_docker_helpers}

19
man/tidy_har.Rd

@ -0,0 +1,19 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/as-data-frame-har.R
\name{tidy_har}
\alias{tidy_har}
\title{Turn a gnHARly HAR object into a tidy data frame (tibble)}
\usage{
tidy_har(.x, include_content = TRUE)
}
\arguments{
\item{include_content}{if \code{TRUE} (the default) the encoded element content will be returned in the data frame}
\item{x}{A \code{harentry} object}
}
\value{
data frame (tibble)
}
\description{
Turn a gnHARly HAR object into a tidy data frame (tibble)
}
Loading…
Cancel
Save