Consider changes in height
attempt to wait until window is available
This commit is contained in:
		
							parent
							
								
									da0ba0a40d
								
							
						
					
					
						commit
						2a7c2fc750
					
				
					 1 changed files with 76 additions and 47 deletions
				
			
		
							
								
								
									
										49
									
								
								scripts/external/scraper.js
									
										
									
									
										vendored
									
									
								
							
							
						
						
									
										49
									
								
								scripts/external/scraper.js
									
										
									
									
										vendored
									
									
								
							|  | @ -3,26 +3,36 @@ Read the contents of the page. | |||
| */ | ||||
| 
 | ||||
| export default class scraper { | ||||
| 	constructor(scraper_fields) { | ||||
| 	/* | ||||
| 	Scrape fields.  | ||||
| 
 | ||||
| 	@param {Object} scraper_fields the fields to scrape | ||||
| 	@param {Object} options the options | ||||
| 	*/ | ||||
| 	constructor(scraper_fields, options = {"wait until available": true}) { | ||||
| 		let field_content; | ||||
| 
 | ||||
| 		// Quickly scroll down then to where the user already was to get automatically hidden content. 
 | ||||
| 		function autoscroll() { | ||||
| 			let SCROLL = {"x": window.scrollX, "y": window.scrollY}; | ||||
| 			let SCROLL = {"x": parseInt(window.scrollX), "y": parseInt(window.scrollY)}; | ||||
| 
 | ||||
| 			// Repeat two times to ensure proper webpage load. 
 | ||||
| 			for (let TIMES = 1; TIMES <= 2; TIMES++) { | ||||
| 				[{"top": 0, "left": 0, "behavior": "smooth"}, {"top": document.body.scrollHeight, "left": document.body.scrollWidth, "behavior": "smooth"}].forEach((POSITION) => { | ||||
| 			// Repeat every ten milliseconds until 3 times.
 | ||||
| 
 | ||||
| 			for (let SCROLLS = 1; SCROLLS <= 2; SCROLLS++) { | ||||
| 				[{"top": document.body.scrollHeight, "left": document.body.scrollWidth}, {"top": 0, "left": 0}].forEach(POSITION => { | ||||
| 					setTimeout(() => { | ||||
| 						window.scrollTo(POSITION); | ||||
| 				}) | ||||
| 			}; | ||||
| 					}, 10); | ||||
| 				}); | ||||
| 			} | ||||
| 
 | ||||
| 			// Scroll back to user's previous position.
 | ||||
| 			window.scrollTo({"top": SCROLL[`y`], "left": SCROLL[`x`], "behavior": "smooth"}); | ||||
| 			setTimeout(() => { | ||||
| 				window.scrollTo(SCROLL); | ||||
| 			}, 5) | ||||
| 		}; | ||||
| 
 | ||||
| 		autoscroll(); | ||||
| 
 | ||||
| 		const read = () => { | ||||
| 			if ((typeof scraper_fields).includes("object") && scraper_fields != null && scraper_fields) { | ||||
| 	 | ||||
| 				/* Read for the particular fields. */ | ||||
|  | @ -67,5 +77,24 @@ export default class scraper { | |||
| 					this[field_name] = field_content[field_name]; | ||||
| 				}); | ||||
| 			} | ||||
| 		}; | ||||
| 
 | ||||
| 		// Check every 1 second to check until autosccroll is done.
 | ||||
| 		function wait_autoscroll(OPTIONS) { | ||||
| 			return new Promise((resolve, reject) => { | ||||
| 				// Check if autoscroll is done.
 | ||||
| 				if (!((typeof window).includes(`undef`))) { | ||||
| 					autoscroll(); | ||||
| 					resolve(); | ||||
| 				} else if (OPTIONS[`wait until available`]) { | ||||
| 					setTimeout(() => { | ||||
| 						wait_autoscroll().then(resolve).catch(reject); | ||||
| 					}, 1000); | ||||
| 				} else { | ||||
| 					reject(); | ||||
| 				} | ||||
| 			}); | ||||
| 		} | ||||
| 		wait_autoscroll(options).then(() => {read();}); | ||||
| 	} | ||||
| } | ||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue