move product analysis to the processor
This commit is contained in:
		
							parent
							
								
									85f9dfd5cf
								
							
						
					
					
						commit
						620be2d291
					
				
					 2 changed files with 79 additions and 60 deletions
				
			
		|  | @ -51,9 +51,6 @@ export default class product { | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	async save() { | 	async save() { | ||||||
| 		// Save the status to the storage. This might be needed since it's also a signalling method. 
 |  | ||||||
| 		await global.write([`sites`, this.URL, `status`], this.status, -1, {"strict": true}); |  | ||||||
| 
 |  | ||||||
| 		// There is only a need to save the data if an update is needed. 
 | 		// There is only a need to save the data if an update is needed. 
 | ||||||
| 		if (Object.hasOwn(this.status, `update`) ? this.status[`update`] : true) {	 | 		if (Object.hasOwn(this.status, `update`) ? this.status[`update`] : true) {	 | ||||||
| 			// Save the snip data. 
 | 			// Save the snip data. 
 | ||||||
|  | @ -63,34 +60,4 @@ export default class product { | ||||||
| 			return((this[`analysis`]) ? global.write([`sites`, this.URL, `analysis`], this.analysis, 1) : false); | 			return((this[`analysis`]) ? global.write([`sites`, this.URL, `analysis`], this.analysis, 1) : false); | ||||||
| 		} | 		} | ||||||
| 	}; | 	}; | ||||||
| 
 |  | ||||||
| 	async analyze(options = {}) { |  | ||||||
| 		// Stop when the data is already analyzed.
 |  | ||||||
| 		if (((this.analysis && this.analysis != undefined) ? !((typeof this.analysis).includes(`obj`) && !Array.isArray(this.analysis)) : true) || this.status[`update`] || ((options && (typeof options).includes(`obj`)) ? options[`override`] : false)) { |  | ||||||
| 			const gemini = (await import(chrome.runtime.getURL("scripts/AI/gemini.js"))).default; |  | ||||||
| 			let analyzer = new gemini (await global.read([`settings`,`analysis`,`api`,`key`]), `gemini-pro`); |  | ||||||
| 			 |  | ||||||
| 			// Add the prompt.
 |  | ||||||
| 			let PROMPT = []; |  | ||||||
| 			PROMPT.push({"text": ((new texts(`AI_message_prompt`)).localized).concat(JSON.stringify(this.details))}); |  | ||||||
| 			 |  | ||||||
| 			// Run the analysis.
 |  | ||||||
| 			await analyzer.generate(PROMPT); |  | ||||||
| 
 |  | ||||||
| 			// Raise an error if the product analysis is blocked. 
 |  | ||||||
| 			this.status[`blocked`] = analyzer.blocked; |  | ||||||
| 			if (this.status[`blocked`]) { |  | ||||||
| 				this.status.error = {"name": (new texts(`blocked`)).localized, "message": (new texts(`error_msg_blocked`)).localized, "stack": analyzer.response}; |  | ||||||
| 				throw new Error((new texts(`error_msg_blocked`)).localized) |  | ||||||
| 			}; |  | ||||||
| 
 |  | ||||||
| 			if (analyzer.candidate) { |  | ||||||
| 				// Remove all markdown formatting.
 |  | ||||||
| 				this.analysis = JSON.parse(analyzer.candidate.replace(/(```json|```|`)/g, '')); |  | ||||||
| 			}; |  | ||||||
| 			 |  | ||||||
| 		}; |  | ||||||
| 
 |  | ||||||
| 		return(this.analysis); |  | ||||||
| 	}; |  | ||||||
| }; | }; | ||||||
|  |  | ||||||
							
								
								
									
										80
									
								
								scripts/external/processor.js
									
										
									
									
										vendored
									
									
								
							
							
						
						
									
										80
									
								
								scripts/external/processor.js
									
										
									
									
										vendored
									
									
								
							|  | @ -4,12 +4,15 @@ Process the information on the website and display it on screen. | ||||||
| 
 | 
 | ||||||
| import scraper from "/scripts/external/scraper.js"; | import scraper from "/scripts/external/scraper.js"; | ||||||
| import product from "/scripts/data/product.js"; | import product from "/scripts/data/product.js"; | ||||||
| import {global} from "/scripts/secretariat.js"; | import {global, observe} from "/scripts/secretariat.js"; | ||||||
| import logging from "/scripts/logging.js"; | import logging from "/scripts/logging.js"; | ||||||
|  | import texts from "/scripts/mapping/read.js"; | ||||||
| import {URLs} from "/scripts/utils/URLs.js"; | import {URLs} from "/scripts/utils/URLs.js"; | ||||||
|  | import gemini from "/scripts/AI/gemini.js"; | ||||||
| 
 | 
 | ||||||
| export default class processor { | export default class processor { | ||||||
| 	#filter;  | 	#filter;  | ||||||
|  | 	#analyzer; | ||||||
| 	 | 	 | ||||||
| 	async scrape (fields, options) { | 	async scrape (fields, options) { | ||||||
| 		this.product.details = new scraper (((fields) ? fields : this.targets), options);  | 		this.product.details = new scraper (((fields) ? fields : this.targets), options);  | ||||||
|  | @ -22,28 +25,57 @@ export default class processor { | ||||||
| 	} | 	} | ||||||
| 	 | 	 | ||||||
| 	async analyze(options = {}) { | 	async analyze(options = {}) { | ||||||
|  | 		const main = async() => { | ||||||
|  | 			// Set up the analyzer.
 | ||||||
|  | 			this.#analyzer = (this.#analyzer) ? this.#analyzer : new gemini (await global.read([`settings`,`analysis`,`api`,`key`]), `gemini-1.5-pro-latest`); | ||||||
|  | 	 | ||||||
| 			// Set up current data of the site, but forget about its previous errored state. 
 | 			// Set up current data of the site, but forget about its previous errored state. 
 | ||||||
| 		delete this.product.status[`error`]; | 			delete this.status[`error`]; | ||||||
| 	 | 	 | ||||||
| 			// Set the completion state to anything else but not 1. 
 | 			// Set the completion state to anything else but not 1. 
 | ||||||
| 		this.product.status[`done`] = (this.product.status[`done`] == 1) ? 0 : this.product.status[`done`]; | 			(this.status[`done`] >= 1) ? this.#notify(0) : false; | ||||||
| 
 | 
 | ||||||
| 		// Save this information. 
 | 			const perform = async() => { | ||||||
| 		this.product.save(); | 				if (((this.product.analysis && !((typeof this.product.analysis).includes(`undef`))) | ||||||
|  | 					? !((typeof this.product.analysis).includes(`obj`) && !Array.isArray(this.product.analysis)) | ||||||
|  | 					: true) | ||||||
|  | 				|| this.product.status[`update`] || ((options && (typeof options).includes(`obj`)) ? options[`override`] : false)) { | ||||||
|  | 					// Add the prompt.
 | ||||||
|  | 					let PROMPT = []; | ||||||
|  | 					PROMPT.push({"text": ((new texts(`AI_message_prompt`)).localized).concat(JSON.stringify(this.product.details.texts))}); | ||||||
|  | 					 | ||||||
|  | 					// Run the analysis.
 | ||||||
|  | 					await this.#analyzer.generate(PROMPT); | ||||||
|  | 	 | ||||||
|  | 					// Raise an error if the product analysis is blocked. 
 | ||||||
|  | 					this.status[`blocked`] = this.#analyzer.blocked; | ||||||
|  | 					if (this.status[`blocked`]) { | ||||||
|  | 						this.status.error = {"name": (new texts(`blocked`)).localized, "message": (new texts(`error_msg_blocked`)).localized, "stack": analyzer.response}; | ||||||
|  | 						throw Error(); | ||||||
|  | 					}; | ||||||
|  | 	 | ||||||
|  | 					if (this.#analyzer.candidate && !this.status[`blocked`]) { | ||||||
|  | 						// Remove all markdown formatting.
 | ||||||
|  | 						this.product.analysis = JSON.parse(this.#analyzer.candidate.replace(/(```json|```|`)/g, '')); | ||||||
|  | 						 | ||||||
|  | 						// Save the data. 
 | ||||||
|  | 						await this.product.save(); | ||||||
|  | 					}; | ||||||
|  | 				} | ||||||
|  | 			} | ||||||
| 	 | 	 | ||||||
| 			// Try analysis of the data.
 | 			// Try analysis of the data.
 | ||||||
| 			try { | 			try { | ||||||
| 			await this.product.analyze(options); | 				await perform(); | ||||||
| 			} catch(err) { | 			} catch(err) { | ||||||
| 			// Use the existing error if there exists any. 
 | 				// Use the existing error, if any exists. 
 | ||||||
| 			this.product.status[`error`] = (this.product.status[`error`]) ? this.product.status[`error`] : {}; | 				(this.status.error) ? false :  | ||||||
| 
 | 				[`name`, `message`, `stack`].forEach((KEY) => { | ||||||
| 			(this.product.status[`error`]) ? false | 					this.status.error = String(err[KEY]); | ||||||
| 			:  [`name`, `message`, `stack`].forEach((KEY) => { |  | ||||||
| 				this.product.status.error[KEY] = String(err[KEY]); |  | ||||||
| 				}); | 				}); | ||||||
|  | 	 | ||||||
|  | 				// Display the error. 
 | ||||||
| 				logging.error(err); | 				logging.error(err); | ||||||
| 			logging.error(this.product.status[`error`].name, this.product.status[`error`].message, this.product.status[`error`].stack); |  | ||||||
| 			}; | 			}; | ||||||
| 	 | 	 | ||||||
| 			// Indicate that the process is done. 
 | 			// Indicate that the process is done. 
 | ||||||
|  | @ -53,6 +85,26 @@ export default class processor { | ||||||
| 			this.product.save(); | 			this.product.save(); | ||||||
| 		}; | 		}; | ||||||
| 
 | 
 | ||||||
|  | 		const wait = async () => { | ||||||
|  | 			let RUN = false; | ||||||
|  | 			if (await global.read([`settings`,`analysis`,`api`,`key`])) { | ||||||
|  | 				console.log(`run`); | ||||||
|  | 				await main(); | ||||||
|  | 				RUN = true; | ||||||
|  | 			} else { | ||||||
|  | 				new logging(texts.localized(`AIkey_message_waiting_title`), texts.localized(`AIkey_message_waiting_body`)) | ||||||
|  | 				observe(async () => { | ||||||
|  | 					if ((!RUN) ? (await global.read([`settings`,`analysis`,`api`,`key`])) : false) { | ||||||
|  | 						await main(); | ||||||
|  | 						RUN = true; | ||||||
|  | 					} | ||||||
|  | 				}) | ||||||
|  | 			} | ||||||
|  | 		} | ||||||
|  | 
 | ||||||
|  | 		wait(); | ||||||
|  | 	}; | ||||||
|  | 
 | ||||||
| 	/* | 	/* | ||||||
| 	Run in the chronological order. Useful when needed to be redone manually.  | 	Run in the chronological order. Useful when needed to be redone manually.  | ||||||
| 	*/ | 	*/ | ||||||
|  | @ -60,7 +112,7 @@ export default class processor { | ||||||
| 		this.#notify((this.targets) ? .25 : 0); | 		this.#notify((this.targets) ? .25 : 0); | ||||||
| 
 | 
 | ||||||
| 		// Scrape the data. 
 | 		// Scrape the data. 
 | ||||||
| 		await this.scrape(); | 		await this.scrape(null, ((typeof options).includes(`obj`) && options) ? options[`scrape`] : null); | ||||||
| 		 | 		 | ||||||
| 		if ((this.product.details) ? Object.keys(this.product.details).length : false) { | 		if ((this.product.details) ? Object.keys(this.product.details).length : false) { | ||||||
| 			// Update the status. 
 | 			// Update the status. 
 | ||||||
|  |  | ||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue