[ja,th,vi,zh] combine "ipa" template functions (#1518)

- handle "ca-ipa" template which has partial list nodes
- extract more "*-ipa" templates

Author: xxyzz

src/wiktextract/extractor/ja/header.py

@@ -52,7 +52,9 @@ def extract_header_nodes(
             or "form-of" in node.attrs.get("class", "")
         ):
             continue
-        if isinstance(node, HTMLNode) and node.tag in ["small", "i"]:
+        if (isinstance(node, HTMLNode) and node.tag in ["small", "i"]) or (
+            isinstance(node, WikiNode) and node.kind == NodeKind.ITALIC
+        ):
             raw_tag = clean_node(wxr, None, node).strip("(): ")
             if raw_tag != "又は" and raw_tag not in raw_tags:
                 # ignore "又は"(or) in "ja-noun" template

src/wiktextract/extractor/ja/sound.py

@@ -58,10 +58,11 @@ def process_sound_template(
 ) -> None:
     if t_node.template_name in ["音声", "audio"]:
         extract_audio_template(wxr, t_node, sounds)
-    elif t_node.template_name in ["IPA", "X-SAMPA", "hi-IPA"]:
+    elif t_node.template_name in [
+        "IPA",
+        "X-SAMPA",
+    ] or t_node.template_name.endswith("-IPA"):
         extract_ipa_template(wxr, t_node, sounds)
-    elif t_node.template_name == "sa-IPA":
-        extract_ipa_list_template(wxr, t_node, sounds)
     elif t_node.template_name == "homophones":
         extract_homophones_template(wxr, t_node, sounds)
     elif t_node.template_name == "ja-pron":
@@ -102,32 +103,35 @@ def extract_ipa_template(
     expanded_node = wxr.wtp.parse(
         wxr.wtp.node_to_wikitext(t_node), expand_all=True
     )
-    sounds.extend(extract_ipa_list_item(wxr, expanded_node))
-
-
-def extract_ipa_list_template(
-    wxr: WiktextractContext, t_node: TemplateNode, sounds: list[Sound]
-):
-    expanded_node = wxr.wtp.parse(
-        wxr.wtp.node_to_wikitext(t_node), expand_all=True
-    )
-    for list_item in expanded_node.find_child_recursively(NodeKind.LIST_ITEM):
-        sounds.extend(extract_ipa_list_item(wxr, list_item))
+    no_list_nodes = []
+    for node in expanded_node.children:
+        if isinstance(node, WikiNode) and node.kind == NodeKind.LIST:
+            for list_item in node.find_child(NodeKind.LIST_ITEM):
+                sounds.extend(extract_ipa_list_item(wxr, list_item))
+        else:
+            no_list_nodes.append(node)
+    if len(no_list_nodes) > 0:
+        tmp_node = WikiNode(NodeKind.ROOT, 0)
+        tmp_node.children = no_list_nodes
+        sounds.extend(extract_ipa_list_item(wxr, tmp_node))
 
 
 def extract_ipa_list_item(
     wxr: WiktextractContext, list_item: WikiNode
 ) -> list[Sound]:
-    raw_tag = ""
+    raw_tags = []
     sounds = []
     for span_tag in list_item.find_html_recursively("span"):
         span_class = span_tag.attrs.get("class", "").split()
         if "qualifier-content" in span_class or "ib-content" in span_class:
-            raw_tag = clean_node(wxr, None, span_tag)
+            for raw_tag in clean_node(wxr, None, span_tag).split(","):
+                raw_tag = raw_tag.strip()
+                if raw_tag != "":
+                    raw_tags.append(raw_tag)
         elif "IPA" in span_class or "SAMPA" in span_class:
-            sound = Sound(ipa=clean_node(wxr, None, span_tag))
-            if raw_tag != "":
-                sound.raw_tags.append(raw_tag)
+            sound = Sound(
+                ipa=clean_node(wxr, None, span_tag), raw_tags=raw_tags
+            )
             if sound.ipa != "":
                 if "SAMPA" in span_class:
                     sound.ipa = f"/{sound.ipa}/"

src/wiktextract/extractor/ja/tags.py

@@ -308,6 +308,8 @@
     "廃用": "obsolete",
     # Template:ja-noun
     "旧字体": "kyūjitai",
+    # Template:ca-verb
+    "現在第一人称単数形": ["first-person", "singular", "present"],
 }
 
 TOPICS = {

src/wiktextract/extractor/th/sound.py

@@ -35,24 +35,25 @@ def extract_sound_section(
 def extract_sound_template(
     wxr: WiktextractContext, base_data: WordEntry, t_node: TemplateNode
 ):
-    if t_node.template_name.lower() in ["ipa", "hi-ipa"]:
+    if t_node.template_name in ["ja-pron", "ja-IPA"]:
+        extract_ja_pron_template(wxr, base_data, t_node)
+    elif t_node.template_name == "th-pron":
+        extract_th_pron_template(wxr, base_data, t_node)
+    elif t_node.template_name == "lo-pron":
+        extract_lo_pron_template(wxr, base_data, t_node)
+    elif t_node.template_name == "zh-pron":
+        extract_zh_pron_template(wxr, base_data, t_node)
+    elif (
+        t_node.template_name.lower() == "ipa"
+        or t_node.template_name.lower().endswith(("-ipa", "-pron"))
+    ):
         extract_ipa_template(wxr, base_data, t_node)
-    elif t_node.template_name.lower() in ["vi-ipa", "vi-pron", "sa-ipa"]:
-        extract_vi_ipa_template(wxr, base_data, t_node)
     elif t_node.template_name == "X-SAMPA":
         extract_x_sampa_template(wxr, base_data, t_node)
     elif t_node.template_name == "enPR":
         extract_enpr_template(wxr, base_data, t_node)
     elif t_node.template_name in ["audio", "Audio", "เสียง"]:
         extract_audio_template(wxr, base_data, t_node)
-    elif t_node.template_name == "th-pron":
-        extract_th_pron_template(wxr, base_data, t_node)
-    elif t_node.template_name == "lo-pron":
-        extract_lo_pron_template(wxr, base_data, t_node)
-    elif t_node.template_name in ["ja-pron", "ja-IPA"]:
-        extract_ja_pron_template(wxr, base_data, t_node)
-    elif t_node.template_name == "zh-pron":
-        extract_zh_pron_template(wxr, base_data, t_node)
     elif t_node.template_name in ["rhymes", "rhyme"]:
         extract_rhymes_template(wxr, base_data, t_node)
     elif t_node.template_name in ["homophones", "homophone", "hmp"]:
@@ -65,48 +66,52 @@ def extract_ipa_template(
     expanded_node = wxr.wtp.parse(
         wxr.wtp.node_to_wikitext(t_node), expand_all=True
     )
-    extract_ipa_list_item(wxr, base_data, expanded_node)
+    no_list_nodes = []
+    for node in expanded_node.children:
+        if isinstance(node, WikiNode) and node.kind == NodeKind.LIST:
+            for list_item in node.find_child(NodeKind.LIST_ITEM):
+                extract_ipa_list_item(wxr, base_data, list_item)
+        else:
+            no_list_nodes.append(node)
+    if len(no_list_nodes) > 0:
+        tmp_node = WikiNode(NodeKind.ROOT, 0)
+        tmp_node.children = no_list_nodes
+        extract_ipa_list_item(wxr, base_data, tmp_node)
     clean_node(wxr, base_data, expanded_node)
 
 
 def extract_ipa_list_item(
     wxr: WiktextractContext, base_data: WordEntry, list_item: WikiNode
 ):
-    raw_tag = ""
+    raw_tags = []
     for italic_node in list_item.find_child(NodeKind.ITALIC):
         # Template:vi-ipa location data
         raw_tag = clean_node(wxr, None, italic_node)
+        if raw_tag != "":
+            raw_tags.append(raw_tag)
     for span_tag in list_item.find_html_recursively("span"):
         span_class = span_tag.attrs.get("class", "").split()
         if "qualifier-content" in span_class or "ib-content" in span_class:
-            raw_tag = clean_node(wxr, None, span_tag)
+            for raw_tag in clean_node(wxr, None, span_tag).split(","):
+                raw_tag = raw_tag.strip()
+                if raw_tag != "":
+                    raw_tags.append(raw_tag)
         elif "IPA" in span_class:
-            sound = Sound(ipa=clean_node(wxr, None, span_tag))
-            if raw_tag != "":
-                sound.raw_tags.append(raw_tag)
-                translate_raw_tags(sound)
+            sound = Sound(
+                ipa=clean_node(wxr, None, span_tag), raw_tags=raw_tags
+            )
             if sound.ipa != "":
+                translate_raw_tags(sound)
                 base_data.sounds.append(sound)
         elif "Latn" in span_class:
-            sound = Sound(roman=clean_node(wxr, None, span_tag))
-            if raw_tag != "":
-                sound.raw_tags.append(raw_tag)
-                translate_raw_tags(sound)
+            sound = Sound(
+                roman=clean_node(wxr, None, span_tag), raw_tags=raw_tags
+            )
             if sound.roman != "":
+                translate_raw_tags(sound)
                 base_data.sounds.append(sound)
 
 
-def extract_vi_ipa_template(
-    wxr: WiktextractContext, base_data: WordEntry, t_node: TemplateNode
-):
-    expanded_node = wxr.wtp.parse(
-        wxr.wtp.node_to_wikitext(t_node), expand_all=True
-    )
-    for list_item in expanded_node.find_child_recursively(NodeKind.LIST_ITEM):
-        extract_ipa_list_item(wxr, base_data, list_item)
-    clean_node(wxr, base_data, expanded_node)
-
-
 def extract_ja_pron_template(
     wxr: WiktextractContext, base_data: WordEntry, t_node: TemplateNode
 ):

src/wiktextract/extractor/vi/sound.py

@@ -38,8 +38,6 @@ def extract_sound_template(
         "audio-for-pron",
     ]:
         extract_pron_audio_template(wxr, base_data, t_node)
-    elif t_node.template_name in ["tyz-IPA", "hi-IPA", "sa-IPA", "san-IPA"]:
-        extract_tyz_ipa_template(wxr, base_data, t_node)
     elif t_node.template_name in ["zh-pron", "zho-pron"]:
         extract_zh_pron_template(wxr, base_data, t_node)
     elif t_node.template_name in ["th-pron", "tha-pron"]:
@@ -202,35 +200,6 @@ def extract_audio_template(
     base_data.sounds.append(sound)
 
 
-def extract_tyz_ipa_template(
-    wxr: WiktextractContext, base_data: WordEntry, t_node: TemplateNode
-):
-    expanded_node = wxr.wtp.parse(
-        wxr.wtp.node_to_wikitext(t_node), expand_all=True
-    )
-    for list in expanded_node.find_child(NodeKind.LIST):
-        for list_item in list.find_child(NodeKind.LIST_ITEM):
-            raw_tag = ""
-            for node in list_item.find_child(NodeKind.ITALIC | NodeKind.LINK):
-                if node.kind == NodeKind.ITALIC:
-                    raw_tag = clean_node(wxr, None, node)
-                elif node.kind == NodeKind.LINK:
-                    clean_node(wxr, base_data, node)
-            for span_tag in list_item.find_html_recursively("span"):
-                class_names = span_tag.attrs.get("class", "").split()
-                if "IPA" in class_names:
-                    sound = Sound(ipa=clean_node(wxr, None, span_tag))
-                    if raw_tag != "":
-                        sound.raw_tags.append(raw_tag)
-                    if sound.ipa != "":
-                        translate_raw_tags(sound)
-                        base_data.sounds.append(sound)
-                elif "label-content" in class_names:
-                    raw_tag = clean_node(wxr, None, span_tag)
-    for link_node in expanded_node.find_child(NodeKind.LINK):
-        clean_node(wxr, base_data, link_node)
-
-
 def extract_ipa_template(
     wxr: WiktextractContext,
     base_data: WordEntry,
@@ -241,23 +210,46 @@ def extract_ipa_template(
     expanded_node = wxr.wtp.parse(
         wxr.wtp.node_to_wikitext(t_node), expand_all=True
     )
+    no_list_nodes = []
+    for node in expanded_node.children:
+        if isinstance(node, WikiNode) and node.kind == NodeKind.LIST:
+            for list_item in node.find_child(NodeKind.LIST_ITEM):
+                extract_ipa_list_item(wxr, base_data, list_item, ipa_class)
+        else:
+            no_list_nodes.append(node)
+    if len(no_list_nodes) > 0:
+        tmp_node = WikiNode(NodeKind.ROOT, 0)
+        tmp_node.children = no_list_nodes
+        extract_ipa_list_item(wxr, base_data, tmp_node, ipa_class)
+    clean_node(wxr, base_data, expanded_node)
+
+
+def extract_ipa_list_item(
+    wxr: WiktextractContext,
+    base_data: WordEntry,
+    list_item: WikiNode,
+    class_name: str,
+):
     raw_tags = []
-    for span_tag in expanded_node.find_html("span"):
-        class_names = span_tag.attrs.get("class", "").split()
-        if "qualifier-content" in class_names or "label-content" in class_names:
-            raw_tag = clean_node(wxr, None, span_tag)
-            if raw_tag != "":
-                raw_tags.append(raw_tag)
-        elif ipa_class in class_names:
-            ipa = clean_node(wxr, None, span_tag)
-            if ipa != "":
-                sound = Sound(ipa=ipa, raw_tags=raw_tags)
+    for italic_node in list_item.find_child(NodeKind.ITALIC):
+        raw_tag = clean_node(wxr, None, italic_node)
+        if raw_tag != "":
+            raw_tags.append(raw_tag)
+    for span_tag in list_item.find_html_recursively("span"):
+        span_class = span_tag.attrs.get("class", "").split()
+        if "qualifier-content" in span_class or "label-content" in span_class:
+            for raw_tag in clean_node(wxr, None, span_tag).split(","):
+                raw_tag = raw_tag.strip()
+                if raw_tag != "":
+                    raw_tags.append(raw_tag)
+        elif class_name in span_class:
+            sound = Sound(
+                ipa=clean_node(wxr, None, span_tag), raw_tags=raw_tags
+            )
+            if sound.ipa != "":
                 translate_raw_tags(sound)
                 base_data.sounds.append(sound)
 
-    for link in expanded_node.find_child(NodeKind.LINK):
-        clean_node(wxr, base_data, link)
-
 
 def extract_rhymes_template(
     wxr: WiktextractContext, base_data: WordEntry, t_node: TemplateNode