Add lbToKg pp action to the scraper (#1337)

pkg/scraper/mapped.go

@@ -462,12 +462,25 @@ func (p *postProcessFeetToCm) Apply(value string, q mappedQuery) string {
 	return strconv.Itoa(int(math.Round(centimeters)))
 }
 
+type postProcessLbToKg bool
+
+func (p *postProcessLbToKg) Apply(value string, q mappedQuery) string {
+	const lb_in_kg = 0.45359237
+	w, err := strconv.ParseFloat(value, 64)
+	if err == nil {
+		w = w * lb_in_kg
+		value = strconv.Itoa(int(math.Round(w)))
+	}
+	return value
+}
+
 type mappedPostProcessAction struct {
 	ParseDate  string                   `yaml:"parseDate"`
 	Replace    mappedRegexConfigs       `yaml:"replace"`
 	SubScraper *mappedScraperAttrConfig `yaml:"subScraper"`
 	Map        map[string]string        `yaml:"map"`
 	FeetToCm   bool                     `yaml:"feetToCm"`
+	LbToKg     bool                     `yaml:"lbToKg"`
 }
 
 func (a mappedPostProcessAction) ToPostProcessAction() (postProcessAction, error) {
@@ -511,6 +524,14 @@ func (a mappedPostProcessAction) ToPostProcessAction() (postProcessAction, error
 		action := postProcessFeetToCm(a.FeetToCm)
 		ret = &action
 	}
+	if a.LbToKg {
+		if found != "" {
+			return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "lbToKg")
+		}
+		found = "lbToKg"
+		action := postProcessLbToKg(a.LbToKg)
+		ret = &action
+	}
 
 	if ret == nil {
 		return nil, errors.New("invalid post-process action")

pkg/scraper/xpath_test.go

@@ -105,7 +105,7 @@ const htmlDoc1 = `
 								<b>Weight:</b>
 							</td>
 							<td class="paramvalue">
-								57
+								126
 							</td>
 						</tr>
 						<tr>
@@ -212,7 +212,6 @@ func makeXPathConfig() mappedPerformerScraperConfig {
 	config.mappedConfig["Piercings"] = makeSimpleAttrConfig(makeCommonXPath("Piercings:") + "/comment()")
 	config.mappedConfig["Details"] = makeSimpleAttrConfig(makeCommonXPath("Details:"))
 	config.mappedConfig["HairColor"] = makeSimpleAttrConfig(makeCommonXPath("Hair Color:"))
-	config.mappedConfig["Weight"] = makeSimpleAttrConfig(makeCommonXPath("Weight:"))
 
 	// special handling for birthdate
 	birthdateAttrConfig := makeSimpleAttrConfig(makeCommonXPath("Date of Birth:"))
@@ -252,7 +251,7 @@ func makeXPathConfig() mappedPerformerScraperConfig {
 
 	config.mappedConfig["Gender"] = genderConfig
 
-	// use fixed for height
+	// use fixed for Country
 	config.mappedConfig["Country"] = mappedScraperAttrConfig{
 		Fixed: "United States",
 	}
@@ -264,6 +263,13 @@ func makeXPathConfig() mappedPerformerScraperConfig {
 	}
 	config.mappedConfig["Height"] = heightConfig
 
+	weightConfig := makeSimpleAttrConfig(makeCommonXPath("Weight:"))
+	weightConvAction := postProcessLbToKg(true)
+	weightConfig.postProcessActions = []postProcessAction{
+		&weightConvAction,
+	}
+	config.mappedConfig["Weight"] = weightConfig
+
 	return config
 }
 
@@ -317,10 +323,10 @@ func TestScrapePerformerXPath(t *testing.T) {
 	const tattoos = "None"
 	const piercings = "<!-- None -->"
 	const gender = "Female"
-	const height = "170"
+	const height = "170" //	5ft7
 	const details = "Some girls are so damn hot that they can get you bent out of shape, and you will not even be mad at them for doing so. Well, tawny blonde Mia Malkova can bend her body into any shape she pleases, and that’s sure to satisfy all of the horny cocks and wet pussies out there. This girl has acrobatic and contortionist abilities that could even twist a pretzel into a new knot, which can be very helpful in the ... arrow_drop_down Some girls are so damn hot that they can get you bent out of shape, and you will not even be mad at them for doing so. Well, tawny blonde Mia Malkova can bend her body into any shape she pleases, and that’s sure to satisfy all of the horny cocks and wet pussies out there. This girl has acrobatic and contortionist abilities that could even twist a pretzel into a new knot, which can be very helpful in the VR Porn movies – trust us. Ankles behind her neck and feet over her back so she can kiss her toes, turned, twisted and gyrating, she can fuck any which way she wants (and that ass!), will surely make you fall in love with this hot Virtual Reality Porn slut, as she is one of the finest of them all. Talking about perfection, maybe it’s all the acrobatic work that keeps it in such gorgeous shape? Who cares really, because you just want to take a big bite out of it and never let go. But it’s not all about the body. Mia’s also got a great smile, which might not sound kinky, but believe us, it is a smile that will heat up your innards and drop your pants. Is it her golden skin, her innocent pink lips or that heart-shaped face? There is just too much good stuff going on with Mia Malkova, which is maybe why these past few years have heaped awards upon awards on this Southern California native. Mia came to VR Bangers for her first VR Porn video, so you know she’s only going for top-notch scenes with top-game performers, men, and women. Better hit up that yoga studio if you ever dream of being able to bang a flexible and talented chick like lady Malkova."
 	const hairColor = "Blonde"
-	const weight = "57"
+	const weight = "57" // 126 lb
 
 	verifyField(t, performerName, performer.Name, "Name")
 	verifyField(t, gender, performer.Gender, "Gender")

ui/v2.5/src/components/Changelog/versions/v070.md

@@ -1,6 +1,7 @@
 ### ✨ New Features
 * Support serving UI from specific directory location.
 * Added details, death date, hair color, and weight to Performers.
+* Added `lbToKg` post-process action for performer scrapers.
 * Added details to Studios.
 * Added [perceptual dupe checker](/sceneDuplicateChecker).
 * Add various `count` filter criteria and sort options.

ui/v2.5/src/docs/en/Scraping.md

@@ -346,19 +346,29 @@ The `Measurements` xpath string will replace `$infoPiece` with `//div[@class="in
 
 Post-processing operations are contained in the `postProcess` key. Post-processing operations are performed in the order they are specified. The following post-processing operations are available:
 * `feetToCm`: converts a string containing feet and inches numbers into centimetres. Looks for up to two separate integers and interprets the first as the number of feet, and the second as the number of inches. The numbers can be separated by any non-numeric character including the `.` character. It does not handle decimal numbers. For example `6.3` and `6ft3.3` would both be interpreted as 6 feet, 3 inches before converting into centimetres.
+* `lbToKg`: converts a string containing lbs to kg.
 * `map`: contains a map of input values to output values. Where a value matches one of the input values, it is replaced with the matching output value. If no value is matched, then value is unmodified.
 
 Example:
 ```yaml
 performer:
   Gender:
-    selector: //div[class="example element"]
+    selector: //div[@class="example element"]
     postProcess:
       - map:
           F: Female
           M: Male
+  Height:
+    selector: //span[@id="height"]
+    postProcess:
+      - feetToCm: true
+  Weight:
+    selector: //span[@id="weight"]
+    postProcess:
+      - lbToKg: true
 ```
 Gets the contents of the selected div element, and sets the returned value to `Female` if the scraped value is `F`; `Male` if the scraped value is `M`.
+Height and weight are extracted from the selected spans and converted to `cm` and `kg`.
 
 * `parseDate`: if present, the value is the date format using go's reference date (2006-01-02). For example, if an example date was `14-Mar-2003`, then the date format would be `02-Jan-2006`. See the [time.Parse documentation](https://golang.org/pkg/time/#Parse) for details. When present, the scraper will convert the input string into a date, then convert it to the string format used by stash (`YYYY-MM-DD`). Strings "Today", "Yesterday" are matched (case insensitive) and converted by the scraper so you don't need to edit/replace them.