CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

new

Browse source
This commit is contained in:
CypherpunkSamurai 2023-02-16 13:25:03 +05:30
commit 9aab6148ba
756 changed files with 155754 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

75
timeago-parser/raw/java/CheckAll.java Normal file
View file

@ -0,0 +1,75 @@
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import java.io.File;
import java.io.FileInputStream;
import java.util.Arrays;
import java.util.Map;
public class CheckAll {
public static void main(String[] args) throws Exception {
int SECONDS = 59, currentSeconds = 0;
int MINUTES = 59, currentMinutes = 0;
int HOURS = 23, currentHours = 0;
int DAYS = 6, currentDays = 0;
int WEEKS = 4, currentWeeks = 0;
int MONTHS = 11, currentMonths = 0;
int YEARS = 12, currentYears = 0;
for (String name : Arrays.asList("seconds", "minutes", "hours", "days", "weeks", "months", "years")) {
JsonObject object = JsonParser.object().from(new FileInputStream(new File("timeago-parser/raw/times/" + name + ".json")));
for (Map.Entry<String, Object> entry : object.entrySet()) {
JsonObject value = (JsonObject) entry.getValue();
final int size = value.keySet().size();
if (size >= 80) {
if (name.equals("seconds")) currentSeconds++;
if (name.equals("minutes")) currentMinutes++;
if (name.equals("hours")) currentHours++;
if (name.equals("days")) currentDays++;
if (name.equals("weeks")) currentWeeks++;
if (name.equals("months")) currentMonths++;
if (name.equals("years")) currentYears++;
} else {
System.err.println("Missing some units in: " + name + " → " + entry.getKey() + " (current size = " + size + ")");
}
String number = entry.getKey().replaceAll("\\D", "");
for (Map.Entry<String, Object> langsKeys : value.entrySet()) {
String lang = langsKeys.getKey();
String langValue = String.valueOf(langsKeys.getValue());
String langValueNumber = langValue.replaceAll("\\D", "");
if (!langValueNumber.equals(number)) {
final String msg = langValueNumber.isEmpty() ? "doesn't contain number" : "different number";
System.out.printf("%-20s[!] %22s: %10s = %s \n", entry.getKey(), msg, lang, langValue);
}
}
}
}
System.out.println("\n\nHow many:\n");
if (currentSeconds == SECONDS) System.out.println("seconds: " + currentSeconds);
else System.out.println("[!] missing seconds: " + currentSeconds);
if (currentMinutes == MINUTES) System.out.println("minutes: " + currentMinutes);
else System.out.println("[!] missing minutes: " + currentMinutes);
if (currentHours == HOURS) System.out.println("hours: " + currentHours);
else System.out.println("[!] missing hours: " + currentHours);
if (currentDays == DAYS) System.out.println("days: " + currentDays);
else System.out.println("[!] missing days: " + currentDays);
if (currentWeeks == WEEKS) System.out.println("weeks: " + currentWeeks);
else System.out.println("[!] missing weeks: " + currentWeeks);
if (currentMonths == MONTHS) System.out.println("months: " + currentMonths);
else System.out.println("[!] missing months: " + currentMonths);
if (currentYears == YEARS) System.out.println("years: " + currentYears);
else System.out.println("[!] missing years: " + currentYears);
}
}

70
timeago-parser/raw/java/GenerateOverview.java Normal file
View file

@ -0,0 +1,70 @@
import com.grack.nanojson.JsonAppendableWriter;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.*;
public class GenerateOverview {
public static void main(String[] args) throws Exception {
Map<String, Map<String, Collection<String>>> outMap = new TreeMap<>(new Comparator<String>() {
@Override
public int compare(String o1, String o2) {
return o1.compareToIgnoreCase(o2);
}
});
for (String unitName : Arrays.asList("seconds", "minutes", "hours", "days", "weeks", "months", "years")) {
JsonObject object = JsonParser.object().from(new FileInputStream(new File("timeago-parser/raw/times/" + unitName + ".json")));
for (Map.Entry<String, Object> timeKeyValue : object.entrySet()) {
JsonObject timeObject = (JsonObject) timeKeyValue.getValue();
for (Map.Entry<String, Object> langsKeyValue : timeObject.entrySet()) {
String langKey = langsKeyValue.getKey();
String langValue = langsKeyValue.getValue().toString();
Map<String, Collection<String>> langUnitsMap;
if (outMap.containsKey(langKey)) {
langUnitsMap = outMap.get(langKey);
} else {
langUnitsMap = new TreeMap<>(Utils.compareByUnitName());
outMap.put(langKey, langUnitsMap);
}
Collection<String> langUnitListValues;
if (langUnitsMap.containsKey(unitName)) {
langUnitListValues = langUnitsMap.get(unitName);
} else {
langUnitListValues = new TreeSet<>(Utils.compareByNumber());
langUnitsMap.put(unitName, langUnitListValues);
}
langUnitListValues.add(langValue);
}
}
}
writeMapTo(outMap, JsonWriter.indent(" ").on(new FileOutputStream(new File("timeago-parser/raw/overview.json"))));
}
public static void writeMapTo(Map<String, Map<String, Collection<String>>> outMap, JsonAppendableWriter out) {
out.object();
for (Map.Entry<String, Map<String, Collection<String>>> langMapEntry : outMap.entrySet()) {
final String langName = langMapEntry.getKey();
out.object(langName);
for (Map.Entry<String, Collection<String>> langValuesEntry : langMapEntry.getValue().entrySet()) {
final String unitName = langValuesEntry.getKey();
out.array(unitName);
for (String timeValue : langValuesEntry.getValue()) out.value(timeValue);
out.end();
}
out.end();
}
out.end().done();
}
}

116
timeago-parser/raw/java/GeneratePatternClasses.java Normal file
View file

@ -0,0 +1,116 @@
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.timeago.PatternsHolder;
import org.schabi.newpipe.extractor.timeago.TimeAgoUnit;
import java.io.*;
import java.util.*;
public class GeneratePatternClasses {
public static void main(String[] args) throws FileNotFoundException, JsonParserException {
final InputStream resourceAsStream =
new FileInputStream("timeago-parser/raw/unique_patterns.json");
final JsonObject from = JsonParser.object().from(resourceAsStream);
final TreeMap<String, Object> map = new TreeMap<>(from);
for (Map.Entry<String, Object> entry : map.entrySet()) {
final String languageCode = entry.getKey().replace('-', '_');
final Map<String, Object> unitsList = (Map<String, Object>) entry.getValue();
final String wordSeparator = (String) unitsList.get("word_separator");
final JsonArray seconds = (JsonArray) unitsList.get("seconds");
final JsonArray minutes = (JsonArray) unitsList.get("minutes");
final JsonArray hours = (JsonArray) unitsList.get("hours");
final JsonArray days = (JsonArray) unitsList.get("days");
final JsonArray weeks = (JsonArray) unitsList.get("weeks");
final JsonArray months = (JsonArray) unitsList.get("months");
final JsonArray years = (JsonArray) unitsList.get("years");
final StringBuilder specialCasesString = new StringBuilder();
specialCasesConstruct(TimeAgoUnit.SECONDS, seconds, specialCasesString);
specialCasesConstruct(TimeAgoUnit.MINUTES, minutes, specialCasesString);
specialCasesConstruct(TimeAgoUnit.HOURS, hours, specialCasesString);
specialCasesConstruct(TimeAgoUnit.DAYS, days, specialCasesString);
specialCasesConstruct(TimeAgoUnit.WEEKS, weeks, specialCasesString);
specialCasesConstruct(TimeAgoUnit.MONTHS, months, specialCasesString);
specialCasesConstruct(TimeAgoUnit.YEARS, years, specialCasesString);
System.out.println("Generating \"" + languageCode + "\" pattern class...");
try (final FileWriter fileOut = new FileWriter(
"timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/patterns/" +
languageCode + ".java")) {
final String test = INFO_CLASS_GENERATED + "\n" +
"\n" +
"package org.schabi.newpipe.extractor.timeago.patterns;\n\n" +
"import org.schabi.newpipe.extractor.timeago.PatternsHolder;\n" +
(specialCasesString.length() > 0 ? "import org.schabi.newpipe.extractor.timeago.TimeAgoUnit;\n" : "") +
"\n" +
"public class " + languageCode + " extends PatternsHolder {\n" +
" private static final String WORD_SEPARATOR = \"" + wordSeparator + "\";\n" +
" private static final String[]\n" +
" SECONDS /**/ = {" + join(seconds) + "},\n" +
" MINUTES /**/ = {" + join(minutes) + "},\n" +
" HOURS /**/ = {" + join(hours) + "},\n" +
" DAYS /**/ = {" + join(days) + "},\n" +
" WEEKS /**/ = {" + join(weeks) + "},\n" +
" MONTHS /**/ = {" + join(months) + "},\n" +
" YEARS /**/ = {" + join(years) + "};\n" +
"\n" +
" private static final " + languageCode + " INSTANCE = new " + languageCode + "();\n" +
"\n" +
" public static " + languageCode + " getInstance() {\n" +
" return INSTANCE;\n" +
" }\n" +
"\n" +
" private " + languageCode + "() {\n" +
" super(WORD_SEPARATOR, SECONDS, MINUTES, HOURS, DAYS, WEEKS, MONTHS, YEARS);\n" +
specialCasesString.toString() +
" }\n" +
"}";
fileOut.write(test);
} catch (IOException e) {
e.printStackTrace();
}
}
}
private static void specialCasesConstruct(TimeAgoUnit unit, JsonArray array, StringBuilder stringBuilder) {
final Iterator<Object> iterator = array.iterator();
while (iterator.hasNext()) {
final Object o = iterator.next();
if (o instanceof JsonObject) {
final JsonObject caseObject = (JsonObject) o;
for (Map.Entry<String, Object> caseEntry : caseObject.entrySet()) {
final int caseAmount = Integer.parseInt(caseEntry.getKey());
final String caseText = (String) caseEntry.getValue();
iterator.remove();
stringBuilder.append(" ")
.append("putSpecialCase(TimeAgoUnit.").append(unit.name())
.append(", \"").append(caseText).append("\"")
.append(", ").append(caseAmount).append(");").append("\n");
}
}
}
}
private static final String INFO_CLASS_GENERATED = "/**/// DO NOT MODIFY THIS FILE MANUALLY\n" +
"/**/// This class was automatically generated by \"GeneratePatternClasses.java\",\n" +
"/**/// modify the \"unique_patterns.json\" and re-generate instead.";
private static String join(List<Object> list) {
final StringBuilder toReturn = new StringBuilder();
for (Object o : list) {
toReturn.append('"').append(o).append('"').append(", ");
}
toReturn.setLength(Math.max(toReturn.length() - 2, 0));
return toReturn.toString();
}
}

29
timeago-parser/raw/java/Utils.java Normal file
View file

@ -0,0 +1,29 @@
import java.util.*;
public class Utils {
static Comparator<String> compareByNumber() {
return new Comparator<String>() {
@Override
public int compare(String o1, String o2) {
return extractInt(o1) - extractInt(o2);
}
private int extractInt(String s) {
String num = s.replaceAll("\\D", "");
return num.isEmpty() ? 0 : Integer.parseInt(num);
}
};
}
static Comparator<Object> compareByUnitName() {
return new Comparator<Object>() {
private final List<String> ORDER = Arrays.asList("seconds", "minutes", "hours", "days", "weeks", "months", "years");
@Override
public int compare(Object o1, Object o2) {
return Integer.compare(ORDER.indexOf(o1.toString()), ORDER.indexOf(o2.toString()));
}
};
}
}