DOC update docs for POS tagging

docs/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 91deb7ba5686071aa3de78f413c95d00
+config: d882ecc20fe535cdee3f894a41a2e101
 tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/_modules/index.html

@@ -291,7 +291,7 @@ <h1>All modules for which code is available</h1>
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/_modules/pycantonese/corpus.html

@@ -637,7 +637,7 @@ <h1>Source code for pycantonese.corpus</h1><div class="highlight"><pre>
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/_modules/pycantonese/jyutping/characters.html

@@ -415,7 +415,7 @@ <h1>Source code for pycantonese.jyutping.characters</h1><div class="highlight"><
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/_modules/pycantonese/jyutping/parse_jyutping.html

@@ -434,7 +434,7 @@ <h1>Source code for pycantonese.jyutping.parse_jyutping</h1><div class="highligh
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/_modules/pycantonese/jyutping/tipa.html

@@ -424,7 +424,7 @@ <h1>Source code for pycantonese.jyutping.tipa</h1><div class="highlight"><pre>
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/_modules/pycantonese/jyutping/yale.html

@@ -557,7 +557,7 @@ <h1>Source code for pycantonese.jyutping.yale</h1><div class="highlight"><pre>
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/_modules/pycantonese/pos_tagging/hkcancor_to_ud.html

@@ -433,7 +433,7 @@ <h1>Source code for pycantonese.pos_tagging.hkcancor_to_ud</h1><div class="highl
 
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
-<span class="sd">    tag : str</span>
+<span class="sd">    tag : str, optional</span>
 <span class="sd">        A tag from the original HKCanCor annotated data.</span>
 <span class="sd">        If not provided or ``None``, this function returns the entire</span>
 <span class="sd">        dictionary of the tagset mapping from HKCanCor to UD.</span>
@@ -466,7 +466,7 @@ <h1>Source code for pycantonese.pos_tagging.hkcancor_to_ud</h1><div class="highl
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/_modules/pycantonese/pos_tagging/tagger.html

@@ -578,7 +578,7 @@ <h1>Source code for pycantonese.pos_tagging.tagger</h1><div class="highlight"><p
 <span class="sd">        Supported options:</span>
 
 <span class="sd">        * ``&quot;hkcancor&quot;``, for the tagset used by the original HKCanCor data.</span>
-<span class="sd">          There are 112 tags, 46 of which are described at</span>
+<span class="sd">          There are over 100 tags, 46 of which are described at</span>
 <span class="sd">          http://compling.hss.ntu.edu.sg/hkcancor/.</span>
 <span class="sd">        * ``&quot;universal&quot;`` (default option), for the Universal Dependencies v2</span>
 <span class="sd">          tagset. There are 17 tags; see</span>
@@ -603,7 +603,7 @@ <h1>Source code for pycantonese.pos_tagging.tagger</h1><div class="highlight"><p
 
 <span class="sd">    Examples</span>
 <span class="sd">    --------</span>
-<span class="sd">    &gt;&gt;&gt; words = [&#39;我&#39;, &#39;噚日&#39;, &#39;買&#39;, &#39;嗰&#39;, &#39;對&#39;, &#39;鞋&#39;, &#39;。&#39;]  # I bought those shoes yesterday.</span>
+<span class="sd">    &gt;&gt;&gt; words = [&#39;我&#39;, &#39;噚日&#39;, &#39;買&#39;, &#39;嗰&#39;, &#39;對&#39;, &#39;鞋&#39;, &#39;。&#39;]  # I bought that pair of shoes yesterday.</span>
 <span class="sd">    &gt;&gt;&gt; pos_tag(words)</span>
 <span class="sd">    [(&#39;我&#39;, &#39;PRON&#39;), (&#39;噚日&#39;, &#39;ADV&#39;), (&#39;買&#39;, &#39;VERB&#39;), (&#39;嗰&#39;, &#39;PRON&#39;), (&#39;對&#39;, &#39;NOUN&#39;), (&#39;鞋&#39;, &#39;NOUN&#39;), (&#39;。&#39;, &#39;PUNCT&#39;)]</span>
 <span class="sd">    &gt;&gt;&gt; pos_tag(words, tagset=&quot;hkcancor&quot;)</span>
@@ -632,7 +632,7 @@ <h1>Source code for pycantonese.pos_tagging.tagger</h1><div class="highlight"><p
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/_modules/pycantonese/stop_words.html

@@ -439,7 +439,7 @@ <h1>Source code for pycantonese.stop_words</h1><div class="highlight"><pre>
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/_modules/pycantonese/word_segmentation.html

@@ -403,7 +403,7 @@ <h1>Source code for pycantonese.word_segmentation</h1><div class="highlight"><pr
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/_modules/pylangacq/chat.html

@@ -2270,7 +2270,7 @@ <h1>Source code for pylangacq.chat</h1><div class="highlight"><pre>
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/_sources/changelog.rst.txt

@@ -41,6 +41,8 @@ Fixed
   (a possible issue on Windows).
 * ``jyutping_to_yale`` and ``parse_jyutping`` now return a null value
   (rather than raise an error) when the input is null.
+* The word segmentation function ``segment`` now strips all whitespace
+  from the input unsegmented string before segmenting it.
 
 Security
 ^^^^^^^^

docs/_sources/pos_tagging.rst.txt

@@ -5,31 +5,51 @@ Part-of-Speech Tagging
 
 .. versionadded:: 3.1.0
 
-.. warning::
-    As of November 2020, PyCantonese v3.1.0 hasn't been released yet.
-    All functionality related to part-of-speech tagging
-    is available only through the GitHub repository for early testers.
-    Everything (what functions are provided, how they behave) is subject to
-    change while it is still under active development.
-    To download and install this (unstable) version of PyCantonese
-    (the ``--pre`` flag means a pre-release version):
-
-    .. code-block:: bash
-
-        $ pip install --pre --upgrade pycantonese
-
-    Running ``import pycantonese as pc; print(pc.__version__)`` will show a
-    version number similar to ``3.1.0.dev2``.
-
-    If you notice any issues, please don't hesitate to
-    `report them <https://pycantonese.org/#links>`_.
-
-    While the documentation below is minimal for now, it is going to be updated
-    and expanded once the part-of-speech tagging functionality is finalized
-    in a new PyCantonese release.
-
-:func:`~pycantonese.pos_tag`
-tags words in a segmented sentence or phrase for their parts of speech.
-
-:func:`~pycantonese.pos_tagging.hkcancor_to_ud`
-maps a part-of-speech tag from HKCanCor to Universal Dependencies v2.
+A basic part-of-speech tagger is provided by :func:`~pycantonese.pos_tag`,
+which takes a segmented phrase or sentence as the input:
+
+.. code-block:: python
+
+    >>> import pycantonese as pc
+    >>> unsegmented = '我噚日買嗰對鞋。'  # I bought that pair of shoes yesterday.
+    >>> segmented = pc.segment(unsegmented)
+    ['我', '噚日', '買', '嗰', '對', '鞋', '。']
+    >>> pc.pos_tag(segmented)
+    [('我', 'PRON'), ('噚日', 'ADV'), ('買', 'VERB'), ('嗰', 'PRON'), ('對', 'NOUN'), ('鞋', 'NOUN'), ('。', 'PUNCT')]
+
+The part-of-speech tagger uses the averaged perceptron model trained on
+HKCanCor data.
+HKCanCor has already been tagged for part-of-speech tags,
+with a tagset of over 100 tags (46 of which are described at
+http://compling.hss.ntu.edu.sg/hkcancor/).
+By default, :func:`~pycantonese.pos_tag` maps the HKCanCor tagset to the
+Universal Dependencies v2 tagset
+(with 17 tags, https://universaldependencies.org/u/pos/index.html),
+for cross-linguistic natural language processing work.
+If you would like the original HKCanCor tagset,
+:func:`~pycantonese.pos_tag` accepts the keyword argument ``tagset``:
+
+.. code-block:: python
+
+    >>> pc.pos_tag(segmented, tagset="hkcancor")
+    [('我', 'R'), ('噚日', 'T'), ('買', 'V'), ('嗰', 'R'), ('對', 'Q'), ('鞋', 'N'), ('。', '。')]
+
+The helper function :func:`~pycantonese.pos_tagging.hkcancor_to_ud`
+exposes the tagset mapping from HKCanCor to Universal Dependencies.
+
+Due to the statistical nature of part-of-speech tagging,
+the quality of results from :func:`~pycantonese.pos_tag` depends on
+(i) the training data,
+(ii) the quality of word segmentation, since the function expects a *segmented* input.
+Currently, a major limitation is the fact that HKCanCor is perhaps still
+the only Cantonese corpus with a permissive license that comes annotated
+with part-of-speech tags.
+Its relatively small size (about 150,000 tagged words) means that models
+more sophisticated than a standard averaged perceptron approach wouldn't be worth it.
+If you think the results from :func:`~pycantonese.pos_tag` are odd,
+it is potentially due to the HKCanCor training data
+(e.g., specific occurrences of word + tag combinations might have thrown off the tagger),
+or the quality of word segmentation, especially if your segmented input comes from
+:func:`~pycantonese.segment` (also trained by HKCanCor)
+-- please `get in touch <https://pycantonese.org/index.html#links>`_
+if you would like further investigation.

docs/api.html

@@ -368,7 +368,7 @@ <h2>Natural Language Processing<a class="headerlink" href="#natural-language-pro
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/changelog.html

@@ -311,6 +311,8 @@ <h3>Fixed<a class="headerlink" href="#fixed" title="Permalink to this headline">
 (a possible issue on Windows).</p></li>
 <li><p><code class="docutils literal notranslate"><span class="pre">jyutping_to_yale</span></code> and <code class="docutils literal notranslate"><span class="pre">parse_jyutping</span></code> now return a null value
 (rather than raise an error) when the input is null.</p></li>
+<li><p>The word segmentation function <code class="docutils literal notranslate"><span class="pre">segment</span></code> now strips all whitespace
+from the input unsegmented string before segmenting it.</p></li>
 </ul>
 </div>
 <div class="section" id="security">
@@ -540,7 +542,7 @@ <h2>[0.1] - 2014-12-17<a class="headerlink" href="#id28" title="Permalink to thi
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/data.html

@@ -349,7 +349,7 @@ <h2>Accessing Custom Data<a class="headerlink" href="#accessing-custom-data" tit
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/generated/pycantonese.characters_to_jyutping.html

@@ -337,7 +337,7 @@ <h1>pycantonese.characters_to_jyutping<a class="headerlink" href="#pycantonese-c
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/generated/pycantonese.corpus.CantoneseCHATReader.html

@@ -571,7 +571,7 @@ <h1>pycantonese.corpus.CantoneseCHATReader<a class="headerlink" href="#pycantone
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/generated/pycantonese.corpus.CantoneseCHATReader.search.html

@@ -366,7 +366,7 @@ <h1>pycantonese.corpus.CantoneseCHATReader.search<a class="headerlink" href="#py
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/generated/pycantonese.hkcancor.html

@@ -311,7 +311,7 @@ <h1>pycantonese.hkcancor<a class="headerlink" href="#pycantonese-hkcancor" title
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/generated/pycantonese.jyutping_to_tipa.html

@@ -332,7 +332,7 @@ <h1>pycantonese.jyutping_to_tipa<a class="headerlink" href="#pycantonese-jyutpin
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/generated/pycantonese.jyutping_to_yale.html

@@ -348,7 +348,7 @@ <h1>pycantonese.jyutping_to_yale<a class="headerlink" href="#pycantonese-jyutpin
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/generated/pycantonese.parse_jyutping.html

@@ -329,7 +329,7 @@ <h1>pycantonese.parse_jyutping<a class="headerlink" href="#pycantonese-parse-jyu
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/generated/pycantonese.pos_tag.html

@@ -294,7 +294,7 @@ <h1>pycantonese.pos_tag<a class="headerlink" href="#pycantonese-pos-tag" title="
 Supported options:</p>
 <ul class="simple">
 <li><p><code class="docutils literal notranslate"><span class="pre">&quot;hkcancor&quot;</span></code>, for the tagset used by the original HKCanCor data.
-There are 112 tags, 46 of which are described at
+There are over 100 tags, 46 of which are described at
 <a class="reference external" href="http://compling.hss.ntu.edu.sg/hkcancor/">http://compling.hss.ntu.edu.sg/hkcancor/</a>.</p></li>
 <li><p><code class="docutils literal notranslate"><span class="pre">&quot;universal&quot;</span></code> (default option), for the Universal Dependencies v2
 tagset. There are 17 tags; see
@@ -324,7 +324,7 @@ <h1>pycantonese.pos_tag<a class="headerlink" href="#pycantonese-pos-tag" title="
 </dd>
 </dl>
 <p class="rubric">Examples</p>
-<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">words</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;我&#39;</span><span class="p">,</span> <span class="s1">&#39;噚日&#39;</span><span class="p">,</span> <span class="s1">&#39;買&#39;</span><span class="p">,</span> <span class="s1">&#39;嗰&#39;</span><span class="p">,</span> <span class="s1">&#39;對&#39;</span><span class="p">,</span> <span class="s1">&#39;鞋&#39;</span><span class="p">,</span> <span class="s1">&#39;。&#39;</span><span class="p">]</span>  <span class="c1"># I bought those shoes yesterday.</span>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">words</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;我&#39;</span><span class="p">,</span> <span class="s1">&#39;噚日&#39;</span><span class="p">,</span> <span class="s1">&#39;買&#39;</span><span class="p">,</span> <span class="s1">&#39;嗰&#39;</span><span class="p">,</span> <span class="s1">&#39;對&#39;</span><span class="p">,</span> <span class="s1">&#39;鞋&#39;</span><span class="p">,</span> <span class="s1">&#39;。&#39;</span><span class="p">]</span>  <span class="c1"># I bought that pair of shoes yesterday.</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">pos_tag</span><span class="p">(</span><span class="n">words</span><span class="p">)</span>
 <span class="go">[(&#39;我&#39;, &#39;PRON&#39;), (&#39;噚日&#39;, &#39;ADV&#39;), (&#39;買&#39;, &#39;VERB&#39;), (&#39;嗰&#39;, &#39;PRON&#39;), (&#39;對&#39;, &#39;NOUN&#39;), (&#39;鞋&#39;, &#39;NOUN&#39;), (&#39;。&#39;, &#39;PUNCT&#39;)]</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">pos_tag</span><span class="p">(</span><span class="n">words</span><span class="p">,</span> <span class="n">tagset</span><span class="o">=</span><span class="s2">&quot;hkcancor&quot;</span><span class="p">)</span>
@@ -356,7 +356,7 @@ <h1>pycantonese.pos_tag<a class="headerlink" href="#pycantonese-pos-tag" title="
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/generated/pycantonese.pos_tagging.hkcancor_to_ud.html

@@ -293,7 +293,7 @@ <h1>pycantonese.pos_tagging.hkcancor_to_ud<a class="headerlink" href="#pycantone
 <dl class="field-list simple">
 <dt class="field-odd">Parameters</dt>
 <dd class="field-odd"><dl class="simple">
-<dt><strong>tag</strong><span class="classifier">str</span></dt><dd><p>A tag from the original HKCanCor annotated data.
+<dt><strong>tag</strong><span class="classifier">str, optional</span></dt><dd><p>A tag from the original HKCanCor annotated data.
 If not provided or <code class="docutils literal notranslate"><span class="pre">None</span></code>, this function returns the entire
 dictionary of the tagset mapping from HKCanCor to UD.</p>
 </dd>
@@ -337,7 +337,7 @@ <h1>pycantonese.pos_tagging.hkcancor_to_ud<a class="headerlink" href="#pycantone
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/generated/pycantonese.read_chat.html

@@ -321,7 +321,7 @@ <h1>pycantonese.read_chat<a class="headerlink" href="#pycantonese-read-chat" tit
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>

docs/generated/pycantonese.segment.html

@@ -344,7 +344,7 @@ <h1>pycantonese.segment<a class="headerlink" href="#pycantonese-segment" title="
   <div role="contentinfo">
     <p>
 
-        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 15, 2021
+        &copy; Copyright 2014-2021, Jackson L. Lee | Documentation last updated on February 21, 2021
 
     </p>
   </div>