Webscraping L oop R - PullRequest
       26

Webscraping L oop R

0 голосов
/ 28 апреля 2020

Я пытаюсь создать al oop, который обрабатывает список URL-адресов и собирает желаемый отфильтрованный контент, который затем превращается в отдельные data.frames и где для каждого из них выполняется простой расчет. Если вычисление не может быть выполнено из-за отсутствия данных в этих полях для определенного URL, я хотел бы показать значение «NA». Если желаемый расчет можно выполнить для каждой записи, я бы хотел, чтобы он был проиндексирован по URL-адресу, которому он соответствует, и затем добавлялся строка за строкой в ​​завершенный файл data.frame, который содержит выходные данные для всех записей в исходном списке. Это был бы мой желаемый конечный результат.

Результат пока таков, что мой скрипт запускается и выдает новые переменные, соответствующие их URL, но вычисление всегда "NaN".

Вот моя работа на данный момент:

for (i in urllist3) {
  url <- paste0(urllist3[i])
  df = read.table(url, sep="|")
  form4_2 <- as.character(unlist(df, use.names=FALSE))
  form4_2 <- str_trim(form4_2)
  form4_2 <- paste(form4_2,collapse = "")

  # Px Feed
  Transacted_price <- str_extract_all(form4_2, 
                    "(?<=(<transactionPricePerShare><value>))\\d+\\.\\d+(?=(</value>))")
  df4 <- data.frame(unlist(Transacted_price))
  colnames(df4) <- "Transaction Prices"


  Transacted_shares <- str_extract_all(form4_2, 
                    "(?<=(<transactionShares><value>))\\d+(?=(</value>))")
  df5 <- data.frame(unlist(Transacted_shares))
  colnames(df5) <- "Transaction Share Quantities"

  Transacted_date <- str_extract_all(form4_2, 
                    "(?<=(<transactionDate><value>))\\d+\\-\\d+-\\d+(?=(</value>))")
  df6 <- data.frame(unlist(Transacted_date))
  colnames(df6) <- "Transaction Date"

  df7 <- cbind(df6,df4,df5)

  # Final Weighted Avg Transactions Px
  df8 <-sum(as.numeric(as.character(df7$`Transaction Prices`)) * as.numeric(as.character(df7$`Transaction Share Quantities`)))/
  sum(as.numeric(as.character(df7$`Transaction Share Quantities`)))
  df8 <- data.frame(df8)
  colnames(df8) <- "Avg Transaction Price ($)"


  # df9 <- cbind(paste(all_text3),readHTMLTable(tableNodes[[8]]),web8)
  assign(i, df8)


  # ADD COLUMN TO IDENTIFY STOCK 
  df9 <- get(i)
  df9['Form 4 URL'] <- i
  assign(i, df9)


}

И dput для urllist3:

dput(urllist3)
c("https://www.sec.gov//Archives/edgar/data/1608233/000151028119000153/0001510281-19-000153.txt", 
"https://www.sec.gov//Archives/edgar/data/1608233/000151028119000153/0001510281-19-000153.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000121/0001510281-20-000121.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000119/0001510281-20-000119.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000118/0001510281-20-000118.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000117/0001510281-20-000117.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000118/0001510281-20-000118.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000117/0001510281-20-000117.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000119/0001510281-20-000119.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000115/0001510281-20-000115.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000114/0001510281-20-000114.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000116/0001510281-20-000116.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000114/0001510281-20-000114.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000113/0001510281-20-000113.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000116/0001510281-20-000116.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000115/0001510281-20-000115.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000110/0001510281-20-000110.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000112/0001510281-20-000112.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000111/0001510281-20-000111.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000110/0001510281-20-000110.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000112/0001510281-20-000112.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000107/0001510281-20-000107.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000108/0001510281-20-000108.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000109/0001510281-20-000109.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000107/0001510281-20-000107.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000105/0001510281-20-000105.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000104/0001510281-20-000104.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000106/0001510281-20-000106.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000105/0001510281-20-000105.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000104/0001510281-20-000104.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000102/0001510281-20-000102.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000101/0001510281-20-000101.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000103/0001510281-20-000103.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000100/0001510281-20-000100.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000101/0001510281-20-000101.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000102/0001510281-20-000102.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000099/0001510281-20-000099.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000094/0001510281-20-000094.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000093/0001510281-20-000093.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000094/0001510281-20-000094.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000093/0001510281-20-000093.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000091/0001510281-20-000091.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000090/0001510281-20-000090.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000091/0001510281-20-000091.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000090/0001510281-20-000090.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000092/0001510281-20-000092.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000087/0001510281-20-000087.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000088/0001510281-20-000088.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000089/0001510281-20-000089.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000087/0001510281-20-000087.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000084/0001510281-20-000084.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000085/0001510281-20-000085.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000086/0001510281-20-000086.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000084/0001510281-20-000084.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000079/0001510281-20-000079.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000080/0001510281-20-000080.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000083/0001510281-20-000083.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000081/0001510281-20-000081.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000080/0001510281-20-000080.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000082/0001510281-20-000082.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000075/0001510281-20-000075.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000076/0001510281-20-000076.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000074/0001510281-20-000074.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000077/0001510281-20-000077.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000077/0001510281-20-000077.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000096/0001510281-20-000096.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000071/0001510281-20-000071.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000097/0001510281-20-000097.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000072/0001510281-20-000072.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000095/0001510281-20-000095.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000076/0001510281-20-000076.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000098/0001510281-20-000098.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000073/0001510281-20-000073.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000076/0001510281-20-000076.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000077/0001510281-20-000077.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000075/0001510281-20-000075.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000069/0001510281-20-000069.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000068/0001510281-20-000068.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000067/0001510281-20-000067.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000070/0001510281-20-000070.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000068/0001510281-20-000068.txt", 
"https://www.sec.gov//Archives/edgar/data/1510281/000151028120000067/0001510281-20-000067.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000129680720000002/0001296807-20-000002.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000129680720000001/0001296807-20-000001.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000129680720000001/0001296807-20-000001.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000156761919022698/0001567619-19-022698.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000156761919022698/0001567619-19-022698.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000156761919022698/0001567619-19-022698.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000156761919022698/0001567619-19-022698.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000156761919022698/0001567619-19-022698.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000156761919022698/0001567619-19-022698.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143113020274/0001181431-13-020274.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143113003263/0001181431-13-003263.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143112055298/0001181431-12-055298.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143112052940/0001181431-12-052940.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143112048827/0001181431-12-048827.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143112040928/0001181431-12-040928.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143112023888/0001181431-12-023888.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143112016935/0001181431-12-016935.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143112004758/0001181431-12-004758.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000120919111052653/0001209191-11-052653.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143111048234/0001181431-11-048234.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000120919111040294/0001209191-11-040294.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143111024664/0001181431-11-024664.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143111007597/0001181431-11-007597.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143110053733/0001181431-10-053733.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143110045606/0001181431-10-045606.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143110044783/0001181431-10-044783.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000129680710000003/0001296807-10-000003.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143110024888/0001181431-10-024888.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143110007774/0001181431-10-007774.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109049683/0001181431-09-049683.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109042613/0001181431-09-042613.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109039208/0001181431-09-039208.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109029764/0001181431-09-029764.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109021508/0001181431-09-021508.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109019213/0001181431-09-019213.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109018393/0001181431-09-018393.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109007731/0001181431-09-007731.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143108059006/0001181431-08-059006.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143108051407/0001181431-08-051407.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143108047160/0001181431-08-047160.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143108043085/0001181431-08-043085.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143108024795/0001181431-08-024795.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143108005239/0001181431-08-005239.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143107063144/0001181431-07-063144.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143107055993/0001181431-07-055993.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143107044609/0001181431-07-044609.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143107024497/0001181431-07-024497.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000112528205005060/0001125282-05-005060.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000112528205000588/0001125282-05-000588.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000112528204004583/0001125282-04-004583.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000112528204004583/0001125282-04-004583.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000112528204003737/0001125282-04-003737.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000112528204003247/0001125282-04-003247.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000112528204003246/0001125282-04-003246.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143111007597/0001181431-11-007597.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143107024497/0001181431-07-024497.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143110024888/0001181431-10-024888.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143108043085/0001181431-08-043085.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109018393/0001181431-09-018393.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143110044783/0001181431-10-044783.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143112040928/0001181431-12-040928.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143113020274/0001181431-13-020274.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143110007774/0001181431-10-007774.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143110053733/0001181431-10-053733.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000129680710000002/0001296807-10-000002.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143112004758/0001181431-12-004758.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143108024795/0001181431-08-024795.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109007731/0001181431-09-007731.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109039208/0001181431-09-039208.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000120919111040294/0001209191-11-040294.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143112023888/0001181431-12-023888.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143113003263/0001181431-13-003263.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109049683/0001181431-09-049683.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000120919111052653/0001209191-11-052653.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143108005239/0001181431-08-005239.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143108059006/0001181431-08-059006.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143109029764/0001181431-09-029764.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000156761919022193/0001567619-19-022193.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143111024664/0001181431-11-024664.txt", 
"https://www.sec.gov//Archives/edgar/data/1296807/000118143107044609/0001181431-07-044609.txt"
)

Помощь в решении этого очень ценится!

...