\documentclass[11pt,twoside,a4paper,english]{book}
\usepackage[cjkgb,postscript]{ucs}
\usepackage{babel}
\usepackage[C10,T1]{fontenc}
\usepackage{shortvrb}
\usepackage{ucs}
\usepackage[utf8x]{inputenc}
\usepackage{longtable}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{color}
\usepackage{multirow}
\usepackage{ifthen}
\usepackage[DIV12]{typearea}
% generated by Docutils <http://docutils.sourceforge.net/>
\newlength{\admonitionwidth}
\setlength{\admonitionwidth}{0.9\textwidth}
\newlength{\docinfowidth}
\setlength{\docinfowidth}{0.9\textwidth}
\newlength{\locallinewidth}
\newcommand{\optionlistlabel}[1]{\bf #1 \hfill}
\newenvironment{optionlist}[1]
{\begin{list}{}
  {\setlength{\labelwidth}{#1}
   \setlength{\rightmargin}{1cm}
   \setlength{\leftmargin}{\rightmargin}
   \addtolength{\leftmargin}{\labelwidth}
   \addtolength{\leftmargin}{\labelsep}
   \renewcommand{\makelabel}{\optionlistlabel}}
}{\end{list}}
\newlength{\lineblockindentation}
\setlength{\lineblockindentation}{2.5em}
\newenvironment{lineblock}[1]
{\begin{list}{}
  {\setlength{\partopsep}{\parskip}
   \addtolength{\partopsep}{\baselineskip}
   \topsep0pt\itemsep0.15\baselineskip\parsep0pt
   \leftmargin#1}
 \raggedright}
{\end{list}}
% begin: floats for footnotes tweaking.
\setlength{\floatsep}{0.5em}
\setlength{\textfloatsep}{\fill}
\addtolength{\textfloatsep}{3em}
\renewcommand{\textfraction}{0.5}
\renewcommand{\topfraction}{0.5}
\renewcommand{\bottomfraction}{0.5}
\setcounter{totalnumber}{50}
\setcounter{topnumber}{50}
\setcounter{bottomnumber}{50}
% end floats for footnotes
% some commands, that could be overwritten in the style file.
\newcommand{\rubric}[1]{\subsection*{~\hfill {\it #1} \hfill ~}}
\newcommand{\titlereference}[1]{\textsl{#1}}
% end of "some commands"
% Unicode font:
\usepackage{ttfucs}
\DeclareTruetypeFont{cyberbit}{cyberbit}
% Index:
\usepackage{makeidx}
\makeindex
% Environment for source code listings:
\usepackage{float}
\floatstyle{ruled}
\newfloat{pylisting}{thp}{lop}[chapter]
\floatname{pylisting}{Listing}
% For Python source code:
\usepackage{alltt}
% Python source code: Prompt
\newcommand{\pysrcprompt}[1]{\textbf{#1}}
\newcommand{\pysrcmore}[1]{\textbf{#1}}
% Python source code: Source code
\newcommand{\pysrckeyword}[1]{\textbf{#1}}
\newcommand{\pysrcbuiltin}[1]{\textbf{#1}}
\newcommand{\pysrcstring}[1]{\textit{#1}}
\newcommand{\pysrcother}[1]{\textbf{#1}}
\newcommand{\pysrcdefname}[1]{\textbf{#1}}
% Python source code: Comments
\newcommand{\pysrccomment}[1]{\textrm{#1}}
% Python interpreter: Traceback message
\newcommand{\pysrcexcept}[1]{\textbf{#1}}
% Python interpreter: Output
\newcommand{\pysrcoutput}[1]{#1}
\input{../../doc/definitions.sty}
\usepackage[colorlinks=true,linkcolor=blue,urlcolor=blue,citecolor=blue,bookmarks=true,bookmarksopenlevel=2]{hyperref}
\title{Natural Language Processing}
\author{}
\date{}
\hypersetup{
pdftitle={Natural Language Processing}
}
\raggedbottom
\begin{document}
\maketitle

\setlength{\locallinewidth}{\linewidth}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% CAP abbreviations (map to small caps in LaTeX) 
% Other candidates for global consistency 
% PTB removed since it must be indexed 
% WN removed since it must be indexed 
% misc & punctuation 
% cdots was unicode U+22EF but not working 
% exercise meta-tags 
% Unicode tests 
% phonetic 
% misc 
% used in Unicode section 
% arrows 
% unification stuff 
% Math & Logic 
% sets 
% Greek 
% Chinese 
% People 
% Python example - a snippet of code in running text 
% PlaceHolder example -  something that should be replaced by actual code 
% Linguistic eXample - cited form in running text 
% Emphasized (more declarative than just using *) 
% Grammatical Category - e.g. NP and verb as technical terms 
% Math expression - e.g. especially for variables 
% Textual Math expression - for words 'inside' a math environment 
% Feature (or attribute) 
% Raw LaTeX 
% Raw HTML 
% Feature-value 
% Lexemes 
% Replacements that rely on previous definitions :-) 
\begin{quote}
\begin{description}
\item [Authors:]
Steven Bird, Ewan Klein, Edward Loper


\item [Version:]
0.9.5 (draft only, please send feedback to authors)


\item [Copyright:]
© 2001-2008 the authors


\item [License:]
Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License


\item [Revision:]

\item [Date:]

\end{description}
\end{quote}
\renewcommand{\contentsname}{Contents}
\tableofcontents

\bigskip
\label{contents}
% -*- mode: rst -*- 
% -*- mode: rst -*- 

\setcounter{chapter}{0}
\renewcommand \thechapter{\Roman{chapter}}


%___________________________________________________________________________

\chapter*{Preface}
\label{preface}

{\noindent}
This is a book about \index{Natural Language Processing}\textbf{Natural Language Processing}. By \index{natural
language}\textbf{natural
language} we mean a language that is used for everyday
communication by humans; languages like English, Hindi or
Portuguese. In contrast to artificial languages such as
programming languages and logical formalisms, natural languages have
evolved as they pass from generation to generation, and are hard to
pin down with explicit rules. We will take Natural Language Processing (or NLP
for short) in a wide sense to cover any kind of computer manipulation
of natural language. At one extreme, it could be as simple as counting
the number of times the letter \titlereference{t} occurs in a paragraph of text. At
the other extreme, NLP might involve ``understanding'' complete human
utterances, at least to the extent of being able to give useful
responses to them.

Most human knowledge — and most human communication — is
represented and expressed using language.  Technologies based on NLP
are becoming increasingly widespread. For example,  handheld
computers (PDAs) support predictive text and handwriting recognition;  web
search engines give access to information locked up in unstructured
text; machine translation allows us to retrieve texts written in
Chinese and read them in Spanish.  By providing more natural human-machine interfaces, and more
sophisticated access to stored information, language processing has
come to play a central role in the multilingual information society.

This book provides a comprehensive introduction to the field of NLP.
It can be used for individual study or as the textbook a course
on natural language processing or computational linguistics.
The book is intensely practical, containing
hundreds of fully-worked examples and graded exercises.
It is based on the Python programming language together with an open source
library called the \emph{Natural Language Toolkit} NLTK.
NLTK includes software, data, and documentation, all freely downloadable from \href{http://nltk.org/}{http://nltk.org/}.
Distributions are provided for Windows, Macintosh and Unix platforms.
We encourage you, the reader, to download Python and NLTK, and try out the
examples and exercises along the way.


%___________________________________________________________________________

\section*{Audience}
\label{audience}

This book is intended for a diverse range of people who want to
learn how to write programs that analyze written language:
\begin{quote}
\begin{description}
\item [New to Programming?:]
The book is suitable for readers with no prior
knowledge of programming, and the early chapters contain many examples that
you can simply copy and try for yourself, together with graded exercises.
If you decide you need a more general introduction to Python, we recommend
you read \emph{Learning Python} (O'Reilly) in conjunction with this book.


\item [New to Python?:]
Experienced programmers can quickly learn enough
Python using this book to get immersed in natural language processing.
All relevant Python features are carefully explained and exemplified,
and you will quickly come to appreciate Python's suitability for this
application area.


\item [Already dreaming in Python?:]
Simply skip the Python
introduction, and dig into the interesting language analysis
material that starts in \href{\#chap-words}{Chapter 2}.  Soon you'll
be applying your skills to this exciting new application area.


\end{description}
\end{quote}


%___________________________________________________________________________

\section*{Emphasis}
\label{emphasis}

This book is a \textbf{practical} introduction to NLP.  You will learn by
example, write real programs, and grasp the value of being able to
test an idea through implementation.  If you haven't learnt already,
this book will teach you \textbf{programming}.  Unlike other programming
books, we provide extensive illustrations and exercises from NLP.  The
approach we have taken is also \textbf{principled}, in that we cover the
theoretical underpinnings and don't shy away from careful linguistic
and computational analysis.  We have tried to be \textbf{pragmatic} in
striking a balance between theory and application, and alternate
between the two several times each chapter, identifying the
connections but also the tensions.  Finally, we recognize that you
won't get through this unless it is also \textbf{pleasurable}, so we have
tried to include many applications and examples that are interesting
and entertaining, sometimes whimsical.


%___________________________________________________________________________

\section*{What You Will Learn}
\label{what-you-will-learn}

By digging into the material presented here, you will learn:
\begin{itemize}
\item {} 
how simple programs can help you manipulate and analyze
language data, and how to write these programs;

\item {} 
how key concepts from NLP and linguistics are used to describe and
analyse language;

\item {} 
how data structures and algorithms are used in NLP;

\item {} 
how language data is stored in standard formats, and how data can
be used to evaluate the performance of NLP techniques.

\end{itemize}
% * how linguistic knowledge is used in important language
% technology components;
% * , and skills in algorithmic problem solving, data modeling, and data management;
% * the organization of the field of |NLP|;
% * skills in Python programming for |NLP|. 

Depending on your background, and your motivation for being interested in NLP,
you will gain different kinds of skills and knowledge from this book, as set out below:

\leavevmode
\begin{longtable}[c]{|p{0.156\locallinewidth}|p{0.389\locallinewidth}|p{0.389\locallinewidth}|}
\hline
\multirow{2}{0.16\locallinewidth}{
Goals
} & \multicolumn{2}{l|}{
Background
} \\
\cline{2-2}
\cline{3-3}
 & 
Arts and Humanities
 & 
Science and Engineering
 \\
\hline

Language
Analysis
 & 
Programming to manage
language data, explore
linguistic models, and
test empirical claims
 & 
Language as a source
of interesting problems in
data modeling, data mining,
and knowledge discovery
 \\
\hline

Language
Technology
 & 
Learning to program, with
applications to familiar
problems, to work in language
technology or other technical
field
 & 
Knowledge of linguistic
algorithms and data structures
for high quality, maintainable
language processing software
 \\
\hline
\caption{}\end{longtable}


%___________________________________________________________________________

\section*{Organization}
\label{organization}

The book is structured into three parts, as follows:
\begin{description}
\item[{Part 1: Basics}] \leavevmode 
In this part, we focus on processing text, recognizing and categorizing
words, and how to deal with large amounts of language data.

\item[{Part 2: Parsing}] \leavevmode 
Here, we deal with grammatical structure in text: how words combine to make
phrases and sentences, and how to automatically parse text into
such structures.

\item[{Part 3: Advanced Topics}] \leavevmode 
This final part of the book contains chapters that address selected
topics in NLP in more depth and to a more advanced level.
By design, the chapters in this part can be read independently of
each other.

\end{description}

{\noindent}
The three parts have a common structure: they start off with a
chapter on programming, followed by three chapters on various topics in NLP.
The programming chapters are \emph{foundational}, and you must master this
material before progressing further.

Each chapter consists of an introduction, a sequence of sections
that progress from elementary to advanced material,
and finally a summary and suggestions for further reading.
Most sections include exercises that are graded according to
the following scheme:
☼ is for easy exercises that involve minor modifications
to supplied code samples or other simple activities;
◑ is for intermediate exercises that explore an aspect
of the material in more depth, requiring careful
analysis and design;
★ is for difficult, open-ended tasks that will challenge
your understanding of the material and force you to think independently
(readers new to programming are encouraged to skip these);
☺ is for non-programming exercises for reflection or discussion.
The exercises are important for consolidating the
material in each section, and we strongly encourage you to try a few
before continuing with the rest of the chapter.


%___________________________________________________________________________

\section*{Why Python?}
\label{why-python}

Python is a simple yet powerful programming language with excellent
functionality for processing linguistic data.  Python can be
downloaded for free from \href{http://www.python.org/}{http://www.python.org/}.

Here is a five-line Python program that takes
text input and prints all the words ending in \texttt{ing}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} sys                         \pysrccomment{{\#} load the system library}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} line \pysrckeyword{in} sys.stdin:             \pysrccomment{{\#} for each line of input text}
\pysrcmore{... }    \pysrckeyword{for} word \pysrckeyword{in} line.split():      \pysrccomment{{\#} for each word in the line}
\pysrcmore{... }        \pysrckeyword{if} word.endswith(\pysrcstring{'ing'}):   \pysrccomment{{\#} does the word end in 'ing'?}
\pysrcmore{... }            \pysrckeyword{print} word             \pysrccomment{{\#} if so, print the word}}\end{alltt}

This program illustrates some of the main features of Python.  First,
whitespace is used to \emph{nest} lines of code, thus the line starting
with \texttt{\pysrckeyword{if}} falls inside the scope of the previous line starting with
\texttt{\pysrckeyword{for}}; this ensures that the \texttt{ing} test is performed for each
word.  Second, Python is \emph{object-oriented}; each variable is an entity
that has certain defined attributes and methods.  For example, the
value of the variable \texttt{line} is more than a sequence of characters.
It is a string object that has a \index{method}\textbf{method} (or operation) called
\texttt{split{\linebreak[0]}({\linebreak[0]})} that we can use to break a line into its words.  To apply
a method to an object, we write the object name, followed by a period,
followed by the method name; i.e., \texttt{line{\linebreak[0]}.{\linebreak[0]}split{\linebreak[0]}({\linebreak[0]})}.  Third, methods
have \emph{arguments} expressed inside parentheses.  For instance, in the
example above, \texttt{split{\linebreak[0]}({\linebreak[0]})} had no argument because we were splitting
the string wherever there was white space, and we could therefore use
empty parentheses.  To split a string into sentences delimited by a
period, we would write \texttt{split{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}.{\linebreak[0]}'})}.  Finally, and most importantly,
Python is highly readable, so much so that it is fairly easy to guess
what the above program does even if you have never written a program
before.

We chose Python because it has a shallow learning curve,
its syntax and semantics are transparent,
and it has good string-handling functionality.  As a scripting
language, Python facilitates interactive exploration.  As an
object-oriented language, Python permits data and methods to be
encapsulated and re-used easily.  As a dynamic language, Python
permits attributes to be added to objects on the fly, and permits
variables to be typed dynamically, facilitating rapid development.
Python comes with an extensive
standard library, including components for graphical programming,
numerical processing, and web data processing.

Python is heavily used in industry, scientific research, and education
around the world.  Python is often praised for the way it facilitates
productivity, quality, and maintainability of software.  A collection of
Python success stories is posted at
\href{http://www.python.org/about/success/}{http://www.python.org/about/success/}.

NLTK defines an infrastructure that can be used to build NLP
programs in Python.  It provides
basic classes for representing data relevant to natural language processing;
standard interfaces for performing tasks such as word tokenization,
part-of-speech tagging, and syntactic parsing;
and standard implementations for each task which can be combined to solve complex problems.

NLTK comes with extensive documentation. In addition to the
book you are reading right now, the website \href{http://nltk.org/}{http://nltk.org/} provides
API documentation which covers every module, class and function in the toolkit,
specifying parameters and giving examples of usage.
The website also provides module \index{guides}\textbf{guides}; these contain extensive
examples and test cases, and are intended for users, developers and instructors.


%___________________________________________________________________________

\section*{Learning Python for Natural Language Processing}
\label{learning-python-for-natural-language-processing}

This book contains self-paced learning materials including many examples and exercises.
An effective way to learn is simply to work through the materials.
The program fragments can be copied directly into a Python interactive session.
Any questions concerning the book, or Python and NLP more generally,
can be posted to the NLTK-Users mailing list (see \href{http://nltk.org/}{http://nltk.org/}).
\begin{quote}
\begin{description}
\item [Python Environments:]
The easiest way to start developing Python code, and to run interactive Python
demonstrations, is to use the simple
editor and interpreter GUI that comes with Python called \emph{IDLE}, the \emph{Integrated
DeveLopment Environment for Python}.


\item [NLTK Community:]
NLTK has a large and growing user base.  There are mailing lists for
announcements about NLTK, for developers and for teachers.
\href{http://nltk.org/}{http://nltk.org/} lists many courses around the world where NLTK
and materials from this book have been adopted,
a useful source of extra materials including slides and exercises.


\end{description}
\end{quote}


%___________________________________________________________________________

\section*{The Design of NLTK}
\label{the-design-of-nltk}

NLTK was designed with four primary goals in mind:
\begin{quote}
\begin{description}
\item [Simplicity:]
We have tried to provide an intuitive and appealing
framework along with substantial building blocks, so you can gain
a practical knowledge of NLP without getting bogged down in the
tedious house-keeping usually associated with processing annotated
language data.  We have provided software distributions for
several platforms, along with platform-specific instructions, to
make the toolkit easy to install.


\item [Consistency:]
We have made a significant effort to ensure that all the
data structures and interfaces are consistent, making it easy to
carry out a variety of tasks using a uniform framework.


\item [Extensibility:]
The toolkit easily accommodates new components,
whether those components replicate or extend existing functionality.
Moreover, the toolkit is organized so that it is usually obvious where
extensions would fit into the toolkit's infrastructure.


\item [Modularity:]
The interaction between different components of the
toolkit uses simple, well-defined interfaces.  It is possible to
complete individual projects using small parts of the toolkit,
without needing to understand how they interact with the rest of
the toolkit.  This allows students to learn how to use the toolkit
incrementally throughout a course.  Modularity also makes it easier
to change and extend the toolkit.


\end{description}
\end{quote}
% :Well-Documented: The toolkit comes with substantial documentation,
%    including nomenclature, data structures, and implementations. 

Contrasting with these goals are three non-requirements —
potentially useful features that we have deliberately avoided.  First,
while the toolkit provides a wide range of functions, it is not
intended to be encyclopedic; there should be a wide variety of ways
in which students can extend the toolkit.  Second, while the toolkit
should be efficient enough that students can use their NLP systems to
perform meaningful tasks, it does not need to be highly optimized for
runtime performance; such optimizations often involve more complex
algorithms, and sometimes require the use of programming languages
like C or C++.  This would make the toolkit less accessible and more
difficult to install.  Third, we have tried to avoid clever
programming tricks, since clear implementations are preferable to
ingenious yet indecipherable ones.


%___________________________________________________________________________

\section*{For Instructors}
\label{for-instructors}

Natural Language Processing (NLP) is often taught within the
confines of a single-semester course at advanced undergraduate level
or postgraduate level.  Many instructors have found that it is
difficult to cover both the theoretical and practical sides of the
subject in such a short span of time.  Some courses focus on theory to
the exclusion of practical exercises, and deprive students of the
challenge and excitement of writing programs to automatically process
language.  Other courses are simply designed to teach programming for
linguists, and do not manage to cover any significant NLP content.
NLTK was originally developed to address this problem,
making it feasible to cover a substantial amount of theory and
practice within a single-semester course, even if students have no
prior programming experience.

A significant fraction of any NLP syllabus deals with
algorithms and data structures.  On their own these can be rather
dry, but NLTK brings them to life with the help of
interactive graphical user interfaces making it possible
to view algorithms step-by-step.  Most NLTK components include
a demonstration which performs an interesting task without
requiring any special input from the user.
An effective way to deliver the materials is through interactive
presentation of the examples, entering them in a Python session,
observing what they do, and modifying them to explore some empirical
or theoretical issue.

The book contains hundreds of examples and exercises which can be used
as the basis for student assignments.  The simplest exercises involve
modifying a supplied program fragment in a specified way in order to
answer a concrete question.  At the other end of the spectrum, NLTK
provides a flexible framework for graduate-level research projects,
with standard implementations of all the basic data structures
and algorithms, interfaces to dozens of widely used data-sets (corpora),
and a flexible and extensible architecture.

{\noindent}
We believe this book is unique in providing a comprehensive
framework for students to learn about NLP in the context of learning
to program.  What sets these
materials apart is the tight coupling of the chapters
and exercises with NLTK, giving students — even those with
no prior programming experience — a practical introduction to
NLP.  Once completing these materials, students will be ready to
attempt one of the more advanced textbooks, such as \emph{Speech and
Language Processing}, by Jurafsky and Martin (Prentice Hall, 2008).
\hypertarget{course-plans}{}
\leavevmode
\begin{longtable}[c]{|p{0.470\locallinewidth}|p{0.121\locallinewidth}|p{0.238\locallinewidth}|}
\hline
\multicolumn{3}{|l|}{\textbf{
Suggested Course Plans; Lectures/Lab Sessions per Chapter
}} \\
\hline
\textbf{
Chapter
} & \textbf{
Linguists
} & \textbf{
Computer Scientists
} \\
\hline
\endhead

1 Introduction
 & 
1
 & 
1
 \\
\hline

2 Programming
 & 
4
 & 
1
 \\
\hline

3 Words
 & 
2-3
 & 
2
 \\
\hline

4 Tagging
 & 
2
 & 
2
 \\
\hline

5 Data-Intensive Language Processing
 & 
0-2
 & 
2
 \\
\hline

6 Structured Programming
 & 
2-4
 & 
1
 \\
\hline

7 Chunking
 & 
2
 & 
2
 \\
\hline

8 Grammars and Parsing
 & 
2-6
 & 
2-4
 \\
\hline

9 Advanced Parsing
 & 
1-4
 & 
3
 \\
\hline

10-14 Advanced Topics
 & 
2-8
 & 
2-16
 \\
\hline

Total
 & 
18-36
 & 
18-36
 \\
\hline
\caption{}\end{longtable}


%___________________________________________________________________________

\section*{Acknowledgments}
\label{acknowledgments}

NLTK was originally created as part of a computational linguistics
course in the Department of Computer and Information Science at the
University of Pennsylvania in 2001.  Since then it has been developed
and expanded with the help of dozens of contributors.  It has now been
adopted in courses in dozens of universities, and serves as the basis
of many research projects.

In particular, we're grateful to the following people for their
feedback, comments on earlier drafts, advice, contributions:
Michaela Atterer,
Greg Aumann,
Kenneth Beesley,
Ondrej Bojar,
Trevor Cohn,
Grev Corbett,
James Curran,
Jean Mark Gawron,
Baden Hughes,
Gwillim Law,
Mark Liberman,
Christopher Maloof,
Stefan Müller,
Stuart Robinson,
Jussi Salmela,
Rob Speer.
Many others have contributed to the toolkit, and they are listed
at \href{http://nltk.org/}{http://nltk.org/}.
We are grateful to many colleagues and students for feedback on the text.

We are grateful to the US National Science Foundation, the Linguistic Data Consortium,
and the Universities of Pennsylvania, Edinburgh, and Melbourne for supporting our work
on this book.
% LSA 325: Anna Asbury, Dustin Bowers 


%___________________________________________________________________________

\section*{About the Authors}
\label{about-the-authors}
\begin{figure}[htbp]
\centering

\scalebox{0.437143}{\includegraphics{../images/authors.png}}
\caption{Edward Loper, Ewan Klein, and Steven Bird, Stanford, July 2007}\end{figure}

\textbf{Steven Bird} is Associate Professor in the
Department of Computer Science and Software Engineering
at the University of Melbourne, and Senior Research Associate in the
Linguistic Data Consortium at the University of Pennsylvania.
After completing his undergraduate training in computer
science and mathematics at the University of Melbourne,
Steven went to the University of Edinburgh to study computational linguistics,
and completed his PhD in 1990 under the supervision of Ewan Klein.
He later moved to Cameroon to conduct linguistic fieldwork on the
Grassfields Bantu languages.  More recently, he spent
several years as Associate Director of the Linguistic Data Consortium
where he led an R{\&}D team to create models and tools for large
databases of annotated text.  Back at Melbourne University,
he leads a language technology research group and lectures
in algorithms and Python programming.  Steven is
Vice President of the Association for Computational Linguistics.

\textbf{Ewan Klein} is Professor of Language Technology in the School of
Informatics at the University of Edinburgh. He completed a PhD on
formal semantics at the University of Cambridge in 1978. After some
years working at the Universities of Sussex and Newcastle upon Tyne,
Ewan took up a teaching position at Edinburgh. He was involved in the
establishment of Edinburgh's Language Technology Group 1993, and has
been closely associated with it ever since.  From 2000–2002,
he took leave from the University to act as Research Manager for the
Edinburgh-based Natural Language Research Group of Edify Corporation,
Santa Clara, and was responsible for spoken dialogue processing.  Ewan
is a past President of the European Chapter of the Association for
Computational Linguistics and was a founding member and Coordinator of
the European Network of Excellence in Human Language Technologies
(ELSNET). He has been involved in leading numerous academic-industrial
collaborative projects, the most recent of which is a biological text
mining initiative funded by ITI Life Sciences, Scotland, in
collaboration with Cognia Corporation, NY.

\textbf{Edward Loper} is a doctoral student in the
Department of Computer and Information Sciences
at the University of Pennsylvania, conducting research
on machine learning in natural language processing.
Edward was a student in Steven's graduate course on
computational linguistics in the fall of 2000, and
went on to be a TA and share in the development of
NLTK.  In addition to NLTK, he has
helped develop other major packages for documenting and
testing Python software, \texttt{epydoc} and \texttt{doctest}.


%___________________________________________________________________________
\hspace*{\fill}\hrulefill\hspace*{\fill}

% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}

\part{BASICS}

\textbf{Introduction to Part I}

Part I covers the linguistic and computational analysis of words.
You will learn how to extract the words out of documents and
text collections in multiple languages, automatically
categorize them as nouns, verbs, etc, and access their meanings.
Part I also introduces the required programming skills along
with basic statistical methods.
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% TODO: add exercises for Unicode section? 
% TODO: add bullet points on regular expressions to summary 
% TODO: update cspy reference to more recent book 
% TODO: add pointers to regexp toolkits (e.g. Kodos) 
% TODO: adopt simpler hacker example with only single character transpositions;
% move hacker example to later section (later chapter?) 
\hypertarget{chap-programming}{}
\setcounter{chapter}{0}
\renewcommand \thechapter{\arabic{chapter}}
\setcounter{chapter}{0}

%___________________________________________________________________________

\chapter{Introduction to Natural Language Processing and Python}
\label{introduction-to-natural-language-processing-and-python}
\label{chap-programming}


%___________________________________________________________________________

\section{The Language Challenge}
\label{the-language-challenge}

Today, people from all walks of life — including professionals, students,
and the general population — are confronted by
unprecedented volumes of information, the vast bulk of which is stored
as unstructured text. In 2003, it was estimated that the annual
production of books amounted to 8 Terabytes. (A Terabyte is 1,000
Gigabytes, i.e., equivalent to 1,000 pickup trucks filled with books.)
It would take a human being about five years to read the new
scientific material that is produced every 24 hours.  Although these
estimates are based on printed materials, increasingly the information
is also available electronically. Indeed, there has been an explosion of text
and multimedia content on the World Wide Web.  For many people, a
large and growing fraction of work and leisure time is spent
navigating and accessing this universe of information.
% http://www2.sims.berkeley.edu/courses/is202/f00/lectures/Lecture2.ppt
% William Hayes 

The presence of so much text in electronic form is a huge challenge to
NLP. Arguably, the only way for humans to cope with the information
explosion is to exploit computational techniques that can sift
through huge bodies of text.

Although existing search engines have been crucial to the growth and
popularity of the Web, humans require skill, knowledge, and some luck,
to extract answers to such questions as \docutilsroleexample{What tourist sites can I
visit between Philadelphia and Pittsburgh on a limited budget?}
\docutilsroleexample{What do expert critics say about digital SLR cameras?} \docutilsroleexample{What
predictions about the steel market were made by credible commentators
in the past week?} Getting a computer to answer them automatically
is a realistic long-term goal, but would involve a range of language
processing tasks, including information extraction, inference, and
summarization, and would need to be carried out on a scale and with a
level of robustness that is still beyond our current capabilities.


%___________________________________________________________________________

\subsection{The Richness of Language}
\label{the-richness-of-language}

Language is the chief manifestation of human intelligence.  Through
language we express basic needs and lofty aspirations, technical
know-how and flights of fantasy.  Ideas are shared over great
separations of distance and time.  The following samples from English
illustrate the richness of language:
\begin{itemize}
\item[(1)] \begin{itemize}
\item[a.] 
Overhead the day drives level and grey, hiding the sun by a flight
of grey spears.  (William Faulkner, \emph{As I Lay Dying}, 1935)
\end{itemize}
\begin{itemize}
\item[b.] 
When using the toaster please ensure that the exhaust fan is turned
on. (sign in dormitory kitchen)
\end{itemize}
\begin{itemize}
\item[c.] 
Amiodarone weakly inhibited CYP2C9, CYP2D6, and CYP3A4-mediated
activities with Ki values of 45.1-271.6 μM (Medline, PMID: 10718780)
\end{itemize}
\begin{itemize}
\item[d.] 
Iraqi Head Seeks Arms (spoof news headline)
\end{itemize}
\begin{itemize}
\item[e.] 
The earnest prayer of a righteous man has great power and wonderful
results. (James 5:16b)
\end{itemize}
\begin{itemize}
\item[f.] 
Twas brillig, and the slithy toves did gyre and gimble in the wabe
(Lewis Carroll, \emph{Jabberwocky}, 1872)
\end{itemize}
\begin{itemize}
\item[g.] 
There are two ways to do this, AFAIK :smile:  (internet discussion archive)
\end{itemize}
\end{itemize}

Thanks to this richness, the study of language is part of many
disciplines outside of linguistics, including translation, literary
criticism, philosophy, anthropology and psychology.  Many less obvious
disciplines investigate language use, such as law, hermeneutics,
forensics, telephony, pedagogy, archaeology, cryptanalysis and speech
pathology.  Each applies distinct methodologies to gather
observations, develop theories and test hypotheses.  Yet all serve to
deepen our understanding of language and of the intellect that is
manifested in language.

The importance of language to science and the arts is matched in
significance by the cultural treasure embodied in language.
Each of the world's {\textasciitilde}7,000 human languages is rich in unique respects,
in its oral histories and creation legends, down to its grammatical
constructions and its very words and their nuances of meaning.
Threatened remnant cultures have words to distinguish plant subspecies
according to therapeutic uses that are unknown to science.  Languages
evolve over time as they come into contact with each other and they
provide a unique window onto human pre-history.  Technological change
gives rise to new words like \docutilsroleexample{blog} and new morphemes like \docutilsroleexample{e-} and
\docutilsroleexample{cyber-}.  In many parts of the world, small linguistic variations
from one town to the next add up to a completely different language in
the space of a half-hour drive.  For its breathtaking complexity and
diversity, human language is as a colorful tapestry stretching
through time and space.


%___________________________________________________________________________

\subsection{The Promise of NLP}
\label{the-promise-of-nlp}

As we have seen, NLP is important
for scientific, economic, social, and cultural reasons.  NLP is
experiencing rapid growth as its theories and methods are deployed in
a variety of new language technologies.  For this reason it is
important for a wide range of people to have a working knowledge of
NLP.
Within industry, it includes people in
\index{human-computer interaction}\textit{human-computer interaction}, \index{business information analysis}\textit{business information analysis},
and \index{Web software development}\textit{Web software development}.
Within academia, this includes people in areas from
\index{humanities computing}\textit{humanities computing} and \index{corpus linguistics}\textit{corpus linguistics}
through to \index{computer science}\textit{computer science} and \index{artificial intelligence}\textit{artificial intelligence}.
We hope that you, a member of this diverse
audience reading these materials, will come to appreciate the workings
of this rapidly growing field of NLP and will apply its techniques in
the solution of real-world problems.

This book presents a
carefully-balanced selection of theoretical foundations and practical
applications, and equips readers to work with large datasets, to create
robust models of linguistic phenomena, and to deploy them in working
language technologies.  By integrating all of this into the Natural
Language Toolkit (NLTK), we hope this book opens up the exciting
endeavor of practical natural language processing to a broader
audience than ever before.
% transition to older material; yet to be updated 

The rest of this chapter provides a non-technical overview of Python and will
cover the basic programming knowledge needed for the rest of
the chapters in Part 1.  It contains many examples and exercises;
there is no better way to learn to program than to dive in and try
these yourself.  You should then feel confident in adapting the example for your
own purposes.  Before you know it you will be programming!
\hypertarget{sect-calculator}{}

%___________________________________________________________________________

\section{Computing with Language}
\label{computing-with-language}
\label{sect-calculator}

As we will see, it is easy to get our hands on large quantities of text.
What can we do with it, assuming we can write some simple programs?
Here we will treat the text as data for the programs we write,
programs that manipulate and analyze it in a variety of interesting ways.
The first step is to get started with the Python interpreter.


%___________________________________________________________________________

\subsection{Getting Started}
\label{getting-started}

One of the friendly things about Python is that it allows you
to type directly into the interactive \index{interpreter}\textbf{interpreter} —
the program that will be running your Python programs.
You can run the Python interpreter using a simple graphical interface
called the Interactive DeveLopment Environment (IDLE).
On a Mac you can find this under \texttt{Applications{\linebreak[0]} {\linebreak[0]}-{\linebreak[0]}>{\linebreak[0]} {\linebreak[0]}MacPython},
and on Windows under \texttt{All{\linebreak[0]} {\linebreak[0]}Programs{\linebreak[0]} {\linebreak[0]}-{\linebreak[0]}>{\linebreak[0]} {\linebreak[0]}Python}.
Under Unix you can run Python from the shell by typing \texttt{python}.
The interpreter will print a blurb about your Python version;
simply check that you are running Python 2.4 or greater (here it is 2.5):
\begin{alltt}\small\textbf{Python 2.5 (r25:51918, Sep 19 2006, 08:49:13)
[GCC 4.0.1 (Apple Computer, Inc. build 5341)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
\pysrcprompt{>{}>{}>}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

If you are unable to run the Python interpreter, you probably don't
have Python installed correctly.  Please visit \href{http://nltk.org/}{http://nltk.org/} for
detailed instructions.
\end{boxedminipage}
\end{sffamily}
\end{center}

The \texttt{\pysrcprompt{>{\linebreak[0]}>{\linebreak[0]}>}} prompt indicates that the Python interpreter is now waiting
for input.  Let's begin by using the Python prompt as a calculator:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }1 + 5 * 2 - 3
\pysrcoutput{8}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Once the interpreter has finished calculating the answer and displaying it, the
prompt reappears. This means the Python interpreter is waiting for another instruction.

Try a few more expressions of your own. You can use asterisk (\texttt{*})
for multiplication and slash (\texttt{/}) for division, and parentheses for
bracketing expressions. One strange thing you might come across is
that division doesn't always behave how you expect:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }3/3
\pysrcoutput{1}
\pysrcoutput{}\pysrcprompt{>{}>{}> }1/3
\pysrcoutput{0}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

The second case is surprising because we would expect the answer to
be \texttt{0{\linebreak[0]}.{\linebreak[0]}333333}.
We will come back to why that is the case later on in this chapter.
For now, let's simply observe that these examples demonstrate
how you can work interactively with the
interpreter, allowing you to experiment and explore.
Also, as you will see later, your intuitions about numerical expressions
will be useful for manipulating other kinds of data in Python.

You should also try nonsensical expressions to see how the interpreter handles it:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }1 +
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{  File "<stdin>", line 1}
\pysrcexcept{    1 +}
\pysrcexcept{      {\textasciicircum}}
\pysrcexcept{SyntaxError: invalid syntax}
\pysrcexcept{}\pysrcprompt{>{}>{}>}}\end{alltt}
% The real output doesn't contain the "Traceback ..." line, but
% doctest doesn't recognize it as an exception without it. 

Here we have produced a \index{syntax error}\textbf{syntax error}.  It doesn't make sense
to end an instruction with a plus sign. The Python interpreter indicates
the line where the problem occurred.


%___________________________________________________________________________

\subsection{Searching Text}
\label{searching-text}

Now that we can use the Python interpreter, let's see how we can harness its
power to process text.  The first step is to type a line of magic at the
Python prompt, telling the interpreter to load some texts for us to explore:
\texttt{\pysrckeyword{from} {\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}text{\linebreak[0]} \pysrckeyword{import} {\linebreak[0]}*}.  After printing a welcome message, it loads
the text of several books, including \emph{Moby Dick}.
We can ask the interpreter to give us some information about it, such
as title and word length, by typing \texttt{text1}, and \texttt{len{\linebreak[0]}({\linebreak[0]}text1{\linebreak[0]})}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk.book \pysrckeyword{import} *
\pysrcprompt{>{}>{}> }text1
\pysrcoutput{<Text: Moby Dick by Herman Melville 1851>}}\end{alltt}

We can examine the contents of the book in a variety
of ways.  A concordance view shows us a given word in its context.  Here we
look up the word \docutilsroleexample{monstrous}.  Try seaching for other words; you can use the up-arrow
key to access the previous command and modify the word being searched.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text1.concordance(\pysrcstring{'monstrous'})
\pysrcoutput{mong the former , one was of a most monstrous size . ... This came towards us , o}
\pysrcoutput{ION OF THE PSALMS . " Touching that monstrous bulk of the whale or ork we have re}
\pysrcoutput{all over with a heathenish array of monstrous clubs and spears . Some were thickl}
\pysrcoutput{ed as you gazed , and wondered what monstrous cannibal and savage could ever have}
\pysrcoutput{ that has survived the flood ; most monstrous and most mountainous ! That Himmale}
\pysrcoutput{ they might scout at Moby Dick as a monstrous fable , or still worse and more det}
\pysrcoutput{ath of Radney .'" CHAPTER 55 Of the monstrous Pictures of Whales . I shall ere lo}
\pysrcoutput{ling Scenes . In connexion with the monstrous pictures of whales , I am strongly}}\end{alltt}

You can now try concordance searches on some of the other texts we have included.
For example, to search \emph{Sense and Sensibility} by Jane Austen, for the word
\docutilsroleexample{affection}, use: \texttt{text2{\linebreak[0]}.{\linebreak[0]}concordance{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}affection{\linebreak[0]}'})}.  Search the book of Genesis
to find out how long some people lived, using:
\texttt{text3{\linebreak[0]}.{\linebreak[0]}concordance{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}lived{\linebreak[0]}'})}.  You could look at \texttt{text4}, the
\emph{US Presidential Inaugural Addresses} to see examples of English dating
back to 1789, and search for words like \docutilsroleexample{nation}, \docutilsroleexample{terror}, \docutilsroleexample{god}.
We've also included \texttt{text5}, the \emph{NPS Chat Corpus}: search this for
unconventional words like \docutilsroleexample{im}, \docutilsroleexample{ur}, \docutilsroleexample{lol}.

Once you've spent some time examining these texts, we hope you have a new
sense of the richness and diversity of language.  In the next chapter
you will learn how to access a broader range of text, including text in
languages other than English.

If we can find words in a text, we can also take note of their position within
the text.  We produce a dispersion plot, where each bar represents an instance
of a word and each row represents the entire text.  In \href{\#fig-sensibility}{Figure 1.1}
we see characteristically different roles played by the male and female
protagonists in \emph{Sense and Sensibility}.  In \href{\#fig-inaugural}{Figure 1.2} we
see some striking patterns of word usage over the last 220 years.  You can
produce these plots as shown below.  You might like to try different words,
and different texts.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text2.dispersion{\_}plot({[}\pysrcstring{'Elinor'}, \pysrcstring{'Marianne'}, \pysrcstring{'Edward'}, \pysrcstring{'Willoughby'}{]})
\pysrcprompt{>{}>{}> }text4.dispersion{\_}plot({[}\pysrcstring{'citizens'}, \pysrcstring{'democracy'}, \pysrcstring{'freedom'}, \pysrcstring{'duties'}, \pysrcstring{'America'}{]})}\end{alltt}
\hypertarget{fig-sensibility}{}\begin{figure}[htbp]
\centering

\scalebox{0.514286}{\includegraphics{../images/sensibility.png}}
\caption{Lexical Dispersion Plot for Main Protagonists in \emph{Sense and Sensibility}}\end{figure}
\hypertarget{fig-inaugural}{}\begin{figure}[htbp]
\centering

\scalebox{0.514286}{\includegraphics{../images/inaugural.png}}
\caption{Lexical Dispersion Plot for Words in Presidential Inaugural Addresses}\end{figure}

A concordance permits us to see words in context, e.g. we saw that
\docutilsroleexample{monstrous} appeared in the context \docutilsroleexample{the monstrous pictures}.
What other words appear in the same contexts that \docutilsroleexample{monstrous}
appears in?  We can find out as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text1.similar(\pysrcstring{'monstrous'})
\pysrcoutput{subtly impalpable curious abundant perilous trustworthy untoward}
\pysrcoutput{singular imperial few maddens loving mystifying christian exasperate}
\pysrcoutput{puzzled fearless uncommon domineering candid}
\pysrcoutput{}\pysrcprompt{>{}>{}> }text2.similar(\pysrcstring{'monstrous'})
\pysrcoutput{great very so good vast a exceedingly heartily amazingly as sweet}
\pysrcoutput{remarkably extremely}}\end{alltt}

Observe that we get different results for different books.

Now, just for fun, let's try generating some random text in the various
styles we have just seen.  To do this, we type the name of the text
followed by the ``generate'' function, e.g. \texttt{text3{\linebreak[0]}.{\linebreak[0]}generate{\linebreak[0]}({\linebreak[0]})}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text3.generate()
\pysrcoutput{In the beginning of his brother is a hairy man , whose top may reach}
\pysrcoutput{unto heaven ; and ye shall sow the land of Egypt there was no bread in}
\pysrcoutput{all that he was taken out of the month , upon the earth . So shall thy}
\pysrcoutput{wages be ? And they made their father ; and Isaac was old , and kissed}
\pysrcoutput{him : and Laban with his cattle in the midst of the hands of Esau thy}
\pysrcoutput{first born , and Phichol the chief butler unto his son Isaac , she}}\end{alltt}

Note that first time you run this, it is slow because it gathers statistics
about word sequences.  Each time you run it, you will get different output text.
Now try generating random text in the style of an inaugural address or an
internet chat room.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

When text is printed, punctuation has been split off
from the previous word.  Although this is not correct formatting
for English text, we do this to make it clear that punctuation does
not belong to the word.  This is called ``tokenization'', and we will learn
more about it in \href{\#chap-words}{Chapter 2}.
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\subsection{Counting Vocabulary}
\label{counting-vocabulary}

The most obvious fact about texts that emerges from the previous section is that
they differ in the vocabulary they use.  In this section we will see how to use the
computer to count the words in a text, in a variety of useful ways.
As before you will jump right in and experiment with
the Python interpreter, even though you may not have studied Python systematically
yet.

Let's begin by finding out the length of a text from start to finish,
in terms of the words and punctuation symbols that appear.  Let's look
at the text of \emph{Moby Dick}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }len(text1)
\pysrcoutput{260819}}\end{alltt}

That's a quarter of a million words long!  How many distinct words does this text
contain?  To work this out in Python we have to pose the question slightly
differently.  The vocabulary of a text is just the \emph{set} of words that it uses,
and in Python we can list the vocabulary of \texttt{text3} with the command: \texttt{set{\linebreak[0]}({\linebreak[0]}text3{\linebreak[0]})}.
This will produce many screens of words.  Now try the following:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sorted(set(text3))
\pysrcoutput{{[}'!', "'", '(', ')', ',', ',)', '.', '.)', ':', ';', ';)', '?', '?)',}
\pysrcoutput{'A', 'Abel', 'Abelmizraim', 'Abidah', 'Abide', 'Abimael', 'Abimelech',}
\pysrcoutput{'Abr', 'Abrah', 'Abraham', 'Abram', 'Accad', 'Achbor', 'Adah', ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }len(set(text3))
\pysrcoutput{2789}
\pysrcoutput{}\pysrcprompt{>{}>{}> }len(text3) / len(set(text3))
\pysrcoutput{16}}\end{alltt}

Thus we can see a sorted list of vocabulary items beginning with various
punctuation symbols.  We can find out the size of the vocabulary by asking
for the length of the set.  Finally, we can calculate a measure of the lexical
richness of the text and learn that each word is used 16 times on average.

We might like to repeat the last of these calculations on several texts,
but it is tedious to keep retyping this line for different texts.  Instead,
we can come up with our own name for this task, e.g. ``score'', and
define a function that can be re-used as often as we like:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{score}(text):
\pysrcmore{... }    return len(text) / len(set(text))
\pysrcmore{...}
\pysrcprompt{>{}>{}> }score(text3)
\pysrcoutput{16}
\pysrcoutput{}\pysrcprompt{>{}>{}> }score(text4)
\pysrcoutput{4}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

The Python interpreter changes the prompt from
\texttt{\pysrcprompt{>{\linebreak[0]}>{\linebreak[0]}>}} to \texttt{\pysrcmore{.{\linebreak[0]}.{\linebreak[0]}.}} after encountering the colon at the
end of the first line.  The \texttt{\pysrcmore{.{\linebreak[0]}.{\linebreak[0]}.}} prompt indicates
that Python expects an indented code block to appear next.
It is up to you to do the indentation, by typing four
spaces.  To finish the indented block just enter a blank line.
\end{boxedminipage}
\end{sffamily}
\end{center}

Notice that we used the \texttt{score} function by typing its name, followed
by an open parenthesis, the name of the text, then a close parenthesis.
This is just what we did for the \texttt{len} and \texttt{set} functions earlier.
These parentheses will show up often: their role is to separate
the name of a task — such as \texttt{score} — from the data
that the task is to be performed on — such as \texttt{text3}.

Now that we've had an initial sample of language processing in Python,
we will continue with a systematic introduction to the language.
\hypertarget{sect-basics}{}

%___________________________________________________________________________

\section{Python Basics: Strings and Variables}
\label{python-basics-strings-and-variables}
\label{sect-basics}


%___________________________________________________________________________

\subsection{Representing text}
\label{representing-text}

We can't simply type text directly into the interpreter because it would
try to interpret the text as part of the Python language:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }Hello World
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{  File "<stdin>", line 1}
\pysrcexcept{    Hello World}
\pysrcexcept{              {\textasciicircum}}
\pysrcexcept{SyntaxError: invalid syntax}
\pysrcexcept{}\pysrcprompt{>{}>{}>}}\end{alltt}
% The real output doesn't contain the "Traceback ..." line, but
% doctest doesn't recognize it as an exception without it. 

Here we see an error message.  Note that the interpreter is
confused about the position of the error, and points to the end of
the string rather than the start.

Python represents a piece of text using a \index{string}\textbf{string}.  Strings are
\index{delimited}\textbf{delimited} — or separated from the rest of the program
— by quotation marks:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrcstring{'Hello World'}
\pysrcoutput{'Hello World'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{"Hello World"}
\pysrcoutput{'Hello World'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

We can use either single or double quotation marks, as long as we use the
same ones on either end of the string.

Now we can perform calculator-like operations on strings.
For example, adding two strings together
seems intuitive enough that you could guess the result:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrcstring{'Hello'} + \pysrcstring{'World'}
\pysrcoutput{'HelloWorld'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

When applied to strings, the \texttt{+} operation is called
\index{concatenation}\textbf{concatenation}. It produces a new string that is a copy of the
two original strings pasted together end-to-end.  Notice that
concatenation doesn't do anything clever like insert a space between
the words.  The Python interpreter has no way of knowing that you want
a space; it does \docutilsroleemphasis{exactly} what it is told.  Given the example
of \texttt{+}, you might be able guess what multiplication will do:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrcstring{'Hi'} + \pysrcstring{'Hi'} + \pysrcstring{'Hi'}
\pysrcoutput{'HiHiHi'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'Hi'} * 3
\pysrcoutput{'HiHiHi'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

The point to take from this (apart from learning about strings) is that
in Python, intuition about what should work gets you a long way, so it
is worth just trying things to see what happens.  You are very unlikely
to break anything, so just give it a go.


%___________________________________________________________________________

\subsection{Storing and Reusing Values}
\label{storing-and-reusing-values}

After a while, it can get quite tiresome to keep retyping Python
statements over and over again. It would be nice to be able to store
the \index{value}\textbf{value} of an expression like \texttt{\pysrcstring{'{\linebreak[0]}Hi{\linebreak[0]}'} {\linebreak[0]}+{\linebreak[0]} \pysrcstring{'{\linebreak[0]}Hi{\linebreak[0]}'} {\linebreak[0]}+{\linebreak[0]} \pysrcstring{'{\linebreak[0]}Hi{\linebreak[0]}'}} so that
we can use it again.  We do this by saving results to a location in
the computer's memory, and giving the location a name.  Such a named
place is called a \index{variable}\textbf{variable}.  In Python we create variables by
\index{assignment}\textbf{assignment}, which involves putting a value into the variable:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg = \pysrcstring{'Hello World'}                           ①
\pysrcprompt{>{}>{}> }msg                                           ②
\pysrcoutput{'Hello World'                                     ③}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

In line \href{\#hw-assignment}{①} we have created a variable called \texttt{msg} (short for 'message')
and set it to have the string value \texttt{\pysrcstring{'{\linebreak[0]}Hello{\linebreak[0]} {\linebreak[0]}World{\linebreak[0]}'}}. We used the
\texttt{=} operation, which \index{assigns}\textbf{assigns} the value of the expression on
the right to the variable on the left.  Notice the Python interpreter
does not print any output; it only prints output when the statement
returns a value, and an assignment statement returns no value. In line
\href{\#hw-display}{②} we inspect the contents of the variable by naming it on
the command line: that is, we use the name \texttt{msg}. The interpreter
prints out the contents of the variable in line \href{\#hw-output}{③}.

Variables stand in for values, so instead of writing \texttt{\pysrcstring{'{\linebreak[0]}Hi{\linebreak[0]}'} {\linebreak[0]}*{\linebreak[0]} {\linebreak[0]}3} we
could assign variable \texttt{msg} the value \texttt{\pysrcstring{'{\linebreak[0]}Hi{\linebreak[0]}'}}, and \texttt{num} the
value \texttt{3}, then perform the multiplication using the variable names:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg = \pysrcstring{'Hi'}
\pysrcprompt{>{}>{}> }num = 3
\pysrcprompt{>{}>{}> }msg * num
\pysrcoutput{'HiHiHi'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

The names we choose for the variables are up to us.  Instead of \texttt{msg} and \texttt{num},
we could have used any names we like:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }marta = \pysrcstring{'Hi'}
\pysrcprompt{>{}>{}> }foo123 = 3
\pysrcprompt{>{}>{}> }marta * foo123
\pysrcoutput{'HiHiHi'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent}
Thus, the reason for choosing meaningful variable names is to help you
— and anyone who reads your code — to understand what it is
meant to do.  Python does not try to make sense of the names; it blindly
follows your instructions, and does not object if you do something
potentially confusing such as assigning a variable \texttt{two} the value \texttt{3},
with the assignment statement: \texttt{two{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}3}.

Note that we can also assign a new value to a variable just by using assignment again:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg = msg * num
\pysrcprompt{>{}>{}> }msg
\pysrcoutput{'HiHiHi'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Here we have taken the value of \texttt{msg}, multiplied it by \texttt{3} and then
stored that new string (\texttt{HiHiHi}) back into the variable \texttt{msg}.


%___________________________________________________________________________

\subsection{Printing and Inspecting Strings}
\label{printing-and-inspecting-strings}

So far, when we have wanted to look at the contents of a variable or
see the result of a calculation, we have just typed the variable name
into the interpreter.  We can also see the contents of \texttt{msg} using
\texttt{\pysrckeyword{print} {\linebreak[0]}msg}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg = \pysrcstring{'Hello World'}
\pysrcprompt{>{}>{}> }msg
\pysrcoutput{'Hello World'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{print} msg
\pysrcoutput{Hello World}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

On close inspection, you will see that the quotation marks that
indicate that \texttt{Hello{\linebreak[0]} {\linebreak[0]}World} is a string are missing in the second
case. That is because inspecting a variable, by typing its name into the
interactive interpreter, prints out the Python \index{representation}\textbf{representation} of
a value.  In contrast, the \texttt{\pysrckeyword{print}} statement only prints out the value
itself, which in this case is just the text contained in the string.

In fact, you can use a sequence of comma-separated expressions in a
\texttt{\pysrckeyword{print}} statement:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg2 = \pysrcstring{'Goodbye'}
\pysrcprompt{>{}>{}> }\pysrckeyword{print} msg, msg2
\pysrcoutput{Hello World Goodbye}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

If you have created some variable \texttt{v} and want to find out about
it, then type \texttt{help{\linebreak[0]}({\linebreak[0]}v{\linebreak[0]})} to read the help entry for this kind of object.
Type \texttt{dir{\linebreak[0]}({\linebreak[0]}v{\linebreak[0]})} to see a list of operations that are defined on the
object.
\end{boxedminipage}
\end{sffamily}
\end{center}

You need to be a little bit careful in your choice of names (or \index{identifiers}\textbf{identifiers}) for Python
variables. Some of the things you might try will cause an
error. First, you should start the name with a letter, optionally
followed by digits (\texttt{0} to \texttt{9}) or letters. Thus, \texttt{abc23} is fine, but
\texttt{23abc} will cause a syntax error. You can use underscores (both
within and at the start of the variable name), but not a hyphen, since
this gets interpreted as an arithmetic operator. A second problem is
shown in the following snippet.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{not} = \pysrcstring{"don't do this"}
\pysrcoutput{  File "<stdin>", line 1}
\pysrcoutput{    not = "don't do this"}
\pysrcoutput{    {\textasciicircum}}
\pysrcoutput{SyntaxError: invalid syntax}}\end{alltt}

Why is there an error here? Because \texttt{\pysrckeyword{not}} is reserved as one of Python's 30 odd
\index{keywords}\textbf{keywords}. These are special identifiers that are used in specific
syntactic contexts, and cannot be used as variables. It is easy to
tell which words are keywords if you use IDLE, since they are
helpfully highlighted in orange.


%___________________________________________________________________________

\subsection{Creating Programs with a Text Editor}
\label{creating-programs-with-a-text-editor}

The Python interative interpreter performs your instructions as soon as you type
them.  Often, it is better to compose a multi-line program using a text editor,
then ask Python to run the whole program at once.  Using IDLE, you can do
this by going to the \texttt{File} menu and opening a new window.  Try this now, and
enter the following one-line program:
\begin{quote}{\ttfamily \raggedright \noindent
\small
msg~=~'Hello~World'
}\end{quote}

Save this program in a file called \texttt{test{\linebreak[0]}.{\linebreak[0]}py}, then
go to the \texttt{Run} menu, and select the command \texttt{Run{\linebreak[0]} {\linebreak[0]}Module}.
The result in the main IDLE window should look like this:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }================================ RESTART ================================
\pysrcprompt{>{}>{}>}
\pysrcprompt{>{}>{}>}}\end{alltt}

Now, where is the output showing the value of \texttt{msg}? The answer is
that the program in \texttt{test{\linebreak[0]}.{\linebreak[0]}py} will show a value only if you explicitly tell
it to, using the \texttt{\pysrckeyword{print}} command. So add another line to
\texttt{test{\linebreak[0]}.{\linebreak[0]}py} so that it looks as follows:
\begin{quote}{\ttfamily \raggedright \noindent
\small
msg~=~'Hello~World'~\\
print~msg
}\end{quote}

Select \texttt{Run{\linebreak[0]} {\linebreak[0]}Module} again, and this time you should get output that
looks like this:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }================================ RESTART ================================
\pysrcprompt{>{}>{}>}
\pysrcoutput{Hello World}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

From now on, you have a choice of using the interactive interpreter or a
text editor to create your programs.  It is often convenient to test your ideas
using the interpreter, revising a line of code until it does what you expect,
and consulting the interactive help facility.  Once you're ready, you can paste
the code (minus any \texttt{\pysrcprompt{>{\linebreak[0]}>{\linebreak[0]}>}} prompts) into the text editor,
continue to expand it, and finally save the program
in a file so that you don't have to retype it in again later.


%___________________________________________________________________________

\subsection{Exercises}
\label{exercises}
\newcounter{listcnt0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Start up the Python interpreter (e.g. by running IDLE).
Try the examples in \href{\#sect-calculator}{section 1.2}, then experiment
with using Python as a calculator.

\item {} 
☼ Try the examples in this section, then try the following.
\newcounter{listcnt1}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Create a variable called \texttt{msg} and put a message
of your own in this variable.  Remember that strings need
to be quoted, so you will need to type something like:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg = \pysrcstring{"I like NLP!"}}\end{alltt}

\item {} 
Now print the contents of this variable in two ways, first
by simply typing the variable name and pressing enter, then
by using the \texttt{\pysrckeyword{print}} command.

\item {} 
Try various arithmetic expressions using this string, e.g.
\texttt{msg{\linebreak[0]} {\linebreak[0]}+{\linebreak[0]} {\linebreak[0]}msg}, and \texttt{5{\linebreak[0]} {\linebreak[0]}*{\linebreak[0]} {\linebreak[0]}msg}.

\item {} 
Define a new string \texttt{hello}, and then try \texttt{hello{\linebreak[0]} {\linebreak[0]}+{\linebreak[0]} {\linebreak[0]}msg}.
Change the \texttt{hello} string so that it ends with a space
character, and then try \texttt{hello{\linebreak[0]} {\linebreak[0]}+{\linebreak[0]} {\linebreak[0]}msg} again.

\end{list}

\item {} 
☺ Discuss the steps you would go through to find the ten
most frequent words in a two-page document.

\end{list}


%___________________________________________________________________________

\section{Slicing and Dicing}
\label{slicing-and-dicing}

Strings are so important that we will spend
some more time on them.  Here we will learn how to access the
individual \index{characters}\textbf{characters} that make up a string, how to pull out
arbitrary \index{substrings}\textbf{substrings}, and how to reverse strings.


%___________________________________________________________________________

\subsection{Accessing Individual Characters}
\label{accessing-individual-characters}

The positions within a string are numbered, starting from zero.  To
access a position within a string, we specify the position inside
square brackets:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg = \pysrcstring{'Hello World'}
\pysrcprompt{>{}>{}> }msg{[}0{]}
\pysrcoutput{'H'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }msg{[}3{]}
\pysrcoutput{'l'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }msg{[}5{]}
\pysrcoutput{' '}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

This is called \index{indexing}\textbf{indexing} or \index{subscripting}\textbf{subscripting} the string.
The position we specify inside the square brackets is called the \index{index}\textbf{index}.
We can retrieve not only letters but any character, such as the space at
index \texttt{5}.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Be careful to distinguish between the string \texttt{\pysrcstring{'{\linebreak[0]} {\linebreak[0]}'}}, which
is a single whitespace character, and \texttt{\pysrcstring{'{\linebreak[0]}'}}, which is the empty string.
\end{boxedminipage}
\end{sffamily}
\end{center}

The fact that strings are indexed from zero may seem
counter-intuitive.  You might just want to think of
indexes as giving you the position in a string immediately
\docutilsroleemphasis{before} a character, as indicated in \href{\#indexing01}{Figure 1.3}.
\hypertarget{indexing01}{}\begin{figure}[htbp]
\centering

\scalebox{0.205714}{\includegraphics{../images/indexing01.png}}
\caption{String Indexing}\end{figure}

Now, what happens when we try to access an index that is outside of the string?
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg{[}11{]}
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{  File "<stdin>", line 1, in ?}
\pysrcexcept{IndexError: string index out of range}
\pysrcexcept{}\pysrcprompt{>{}>{}>}}\end{alltt}

The index of \texttt{11} is outside of the range of valid indices (i.e., 0 to 10) for
the string \texttt{\pysrcstring{'{\linebreak[0]}Hello{\linebreak[0]} {\linebreak[0]}World{\linebreak[0]}'}}.
This results in an error message.  This time it is not a syntax error; the
program fragment is syntactically correct.  Instead, the error occurred
while the program was running.  The \texttt{Traceback} message indicates which
line the error occurred on (line 1 of ``standard input'').  It is followed
by the name of the error, \texttt{IndexError}, and a brief explanation.

In general, how do we know what we can index up to?  If we know the
length of the string is \docutilsrolemath{n}, the highest valid index will be \docutilsrolemath{n-1}.  We
can get access to the length of the string using the built-in \texttt{len{\linebreak[0]}({\linebreak[0]})} function.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }len(msg)
\pysrcoutput{11}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Informally, a \index{function}\textbf{function} is a named snippet of code that provides
a service to our program when we \index{call}\textbf{call} or execute it by name.
We call the \texttt{len{\linebreak[0]}({\linebreak[0]})} function by putting parentheses after
the name and giving it the string \texttt{msg} we want to know the length
of. Because \texttt{len{\linebreak[0]}({\linebreak[0]})} is built into the Python interpreter, IDLE colors
it purple.

We have seen what happens when the index is too large.  What about
when it is too small?  Let's see what happens when we use values less
than zero:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg{[}-1{]}
\pysrcoutput{'d'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

This does not generate an error.  Instead, negative indices work from the
\docutilsroleemphasis{end} of the string, so \texttt{-{\linebreak[0]}1} indexes the last character, which is \texttt{\pysrcstring{'{\linebreak[0]}d{\linebreak[0]}'}}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg{[}-3{]}
\pysrcoutput{'r'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }msg{[}-6{]}
\pysrcoutput{' '}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Now the computer works out the location in memory relative to the string's address
plus its length, subtracting the index, e.g. \texttt{3136{\linebreak[0]} {\linebreak[0]}+{\linebreak[0]} {\linebreak[0]}11{\linebreak[0]} {\linebreak[0]}-{\linebreak[0]} {\linebreak[0]}1{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}3146}.
We can also visualize negative indices as shown in \href{\#indexing02}{Figure 1.4}.
\hypertarget{indexing02}{}\begin{figure}[htbp]
\centering

\scalebox{0.205714}{\includegraphics{../images/indexing02.png}}
\caption{Negative Indices}\end{figure}

Thus we have two ways to access the characters in a string, from the
start or the end.  For example, we can access the space in the middle
of \texttt{Hello} and \texttt{World} with either \texttt{msg{\linebreak[0]}{[}{\linebreak[0]}5{\linebreak[0]}{]}} or \texttt{msg{\linebreak[0]}{[}{\linebreak[0]}-{\linebreak[0]}6{\linebreak[0]}{]}};
these refer to the same location, because \texttt{5{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}len{\linebreak[0]}({\linebreak[0]}msg{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}-{\linebreak[0]} {\linebreak[0]}6}.


%___________________________________________________________________________

\subsection{Accessing Substrings}
\label{accessing-substrings}

In NLP we usually want to access more than one character at a time. This is
also pretty simple; we just need to specify a start and end index.  For example,
the following code accesses the substring starting at index \texttt{1}, up to (but not including)
index \texttt{4}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg{[}1:4{]}
\pysrcoutput{'ell'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

The notation \texttt{:{\linebreak[0]}4} is known as a \index{slice}\textbf{slice}.
Here we see the characters are \texttt{\pysrcstring{'{\linebreak[0]}e{\linebreak[0]}'}}, \texttt{\pysrcstring{'{\linebreak[0]}l{\linebreak[0]}'}} and \texttt{\pysrcstring{'{\linebreak[0]}l{\linebreak[0]}'}} which correspond
to \texttt{msg{\linebreak[0]}{[}{\linebreak[0]}1{\linebreak[0]}{]}}, \texttt{msg{\linebreak[0]}{[}{\linebreak[0]}2{\linebreak[0]}{]}} and \texttt{msg{\linebreak[0]}{[}{\linebreak[0]}3{\linebreak[0]}{]}}, but not \texttt{msg{\linebreak[0]}{[}{\linebreak[0]}4{\linebreak[0]}{]}}. This is because
a slice \docutilsroleemphasis{starts} at the first index but finishes \docutilsroleemphasis{one before} the end index.
This is consistent with indexing: indexing also starts from
zero and goes up to \docutilsroleemphasis{one before} the length of the string. We can see this
by slicing with the value of \texttt{len{\linebreak[0]}({\linebreak[0]})}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }len(msg)
\pysrcoutput{11}
\pysrcoutput{}\pysrcprompt{>{}>{}> }msg{[}0:11{]}
\pysrcoutput{'Hello World'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

We can also slice with negative indices — the same basic rule of starting
from the start index and stopping one before the end index applies;
here we stop before the space character:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg{[}0:-6{]}
\pysrcoutput{'Hello'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Python provides two shortcuts for commonly used slice values. If the
start index is \texttt{0} then you can leave it out, and if the
end index is the length of the string then you can leave it out:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg{[}:3{]}
\pysrcoutput{'Hel'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }msg{[}6:{]}
\pysrcoutput{'World'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

The first example above selects the first three characters from the string,
and the second example selects from the character with index 6, namely \texttt{\pysrcstring{'{\linebreak[0]}W{\linebreak[0]}'}},
to the end of the string.


%___________________________________________________________________________

\subsection{Exercises}
\label{id1}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Define a string \texttt{s{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} \pysrcstring{'{\linebreak[0]}colorless{\linebreak[0]}'}}.  Write a Python statement
that changes this to ``colourless'' using only the slice and
concatenation operations.

\item {} 
☼ Try the slice examples from this section using the interactive
interpreter.  Then try some more of your own.  Guess what the result
will be before executing the command.

\item {} 
☼ We can use the slice notation to remove morphological endings on
words.  For example, \texttt{\pysrcstring{'{\linebreak[0]}dogs{\linebreak[0]}'}{[}{\linebreak[0]}:{\linebreak[0]}-{\linebreak[0]}1{\linebreak[0]}{]}} removes the last character of
\texttt{dogs}, leaving \texttt{dog}.  Use slice notation to remove the
affixes from these words (we've inserted a hyphen to
indicate the affix boundary, but omit this from your strings):
\texttt{dish{\linebreak[0]}-{\linebreak[0]}es}, \texttt{run{\linebreak[0]}-{\linebreak[0]}ning}, \texttt{nation{\linebreak[0]}-{\linebreak[0]}ality}, \texttt{un{\linebreak[0]}-{\linebreak[0]}do},
\texttt{pre{\linebreak[0]}-{\linebreak[0]}heat}.

\item {} 
☼ We saw how we can generate an \texttt{IndexError} by indexing beyond the end
of a string.  Is it possible to construct an index that goes too far to
the left, before the start of the string?

\item {} 
☼ We can also specify a ``step'' size for the slice. The following
returns every second character within the slice, in a forward or
reverse direction:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg{[}6:11:2{]}
\pysrcoutput{'Wrd'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }msg{[}10:5:-2{]}
\pysrcoutput{'drW'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Experiment with different step values.

\item {} 
☼ What happens if you ask the interpreter to evaluate \texttt{msg{\linebreak[0]}{[}{\linebreak[0]}:{\linebreak[0]}:{\linebreak[0]}-{\linebreak[0]}1{\linebreak[0]}{]}}?
Explain why this is a reasonable result.

\end{list}


%___________________________________________________________________________

\section{Strings, Sequences, and Sentences}
\label{strings-sequences-and-sentences}

We have seen how words like \docutilsroleexample{Hello} can be stored as a string
\texttt{\pysrcstring{'{\linebreak[0]}Hello{\linebreak[0]}'}}.  Whole sentences can also be stored in strings, and
manipulated as before, as we can see here for Chomsky's \href{http://en.wikipedia.org/wiki/Colorless_green_ideas_sleep_furiously}{famous
nonsense sentence}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = \pysrcstring{'colorless green ideas sleep furiously'}
\pysrcprompt{>{}>{}> }sent{[}16:21{]}
\pysrcoutput{'ideas'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }len(sent)
\pysrcoutput{37}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

However, it turns out to be a bad idea to treat a sentence as a
sequence of its characters, because this makes it too inconvenient to
access the words.  Instead, we would prefer to
represent a sentence as a sequence of its \docutilsroleemphasis{words}; as a result, indexing a
sentence accesses the words, rather than characters.  We will see how to
do this now.


%___________________________________________________________________________

\subsection{Lists}
\label{lists}

A \index{list}\textbf{list} is designed to store a sequence of values.  A list is
similar to a string in many ways except that individual items don't
have to be just characters; they can be arbitrary strings, integers or
even other lists.

A Python list is represented as a sequence of comma-separated items,
delimited by square brackets.  Here are some lists:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }squares = {[}1, 4, 9, 16, 25, 36, 49, 64, 81, 100{]}
\pysrcprompt{>{}>{}> }shopping{\_}list = {[}\pysrcstring{'juice'}, \pysrcstring{'muffins'}, \pysrcstring{'bleach'}, \pysrcstring{'shampoo'}{]}}\end{alltt}

We can also store sentences and phrases using lists.
Let's create part of Chomsky's
sentence as a list and put it in a variable \texttt{cgi}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }cgi = {[}\pysrcstring{'colorless'}, \pysrcstring{'green'}, \pysrcstring{'ideas'}{]}
\pysrcprompt{>{}>{}> }cgi
\pysrcoutput{{[}'colorless', 'green', 'ideas'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Because lists and strings are both kinds of sequence, they can be
processed in similar ways; just as strings support \texttt{len{\linebreak[0]}({\linebreak[0]})}, indexing and
slicing, so do lists. The following example
applies these familiar  operations to the list \texttt{cgi}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }len(cgi)
\pysrcoutput{3}
\pysrcoutput{}\pysrcprompt{>{}>{}> }cgi{[}0{]}
\pysrcoutput{'colorless'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }cgi{[}-1{]}
\pysrcoutput{'ideas'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }cgi{[}-5{]}
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{  File "<stdin>", line 1, in ?}
\pysrcexcept{IndexError: list index out of range}
\pysrcexcept{}\pysrcprompt{>{}>{}>}}\end{alltt}

Here, \texttt{cgi{\linebreak[0]}{[}{\linebreak[0]}-{\linebreak[0]}5{\linebreak[0]}{]}} generates an
error, because the fifth-last item in a three item list would
occur before the list started, i.e., it is undefined.
We can also slice lists in exactly the same way as strings:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }cgi{[}1:3{]}
\pysrcoutput{{[}'green', 'ideas'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }cgi{[}-2:{]}
\pysrcoutput{{[}'green', 'ideas'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Lists can be concatenated just like strings.
Here we will put the resulting list into a new variable \texttt{chomsky}.
The original variable \texttt{cgi} is not changed in the process:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }chomsky = cgi + {[}\pysrcstring{'sleep'}, \pysrcstring{'furiously'}{]}
\pysrcprompt{>{}>{}> }chomsky
\pysrcoutput{{[}'colorless', 'green', 'ideas', 'sleep', 'furiously'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }cgi
\pysrcoutput{{[}'colorless', 'green', 'ideas'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Now, lists and strings do not have exactly the same functionality.
Lists have the added power that you can change their elements. Let's
imagine that we want to change the 0th element of \texttt{cgi} to
\texttt{\pysrcstring{'{\linebreak[0]}colorful{\linebreak[0]}'}}, we can do that by assigning the new value
to the index \texttt{cgi{\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}{]}}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }cgi{[}0{]} = \pysrcstring{'colorful'}
\pysrcprompt{>{}>{}> }cgi
\pysrcoutput{{[}'colorful', 'green', 'ideas'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent} On the other hand if we try to do that with a \emph{string}
— changing the 0th character in \texttt{msg} to \texttt{\pysrcstring{'{\linebreak[0]}J{\linebreak[0]}'}} — we get:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }msg{[}0{]} = \pysrcstring{'J'}
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{  File "<stdin>", line 1, in ?}
\pysrcexcept{TypeError: object does not support item assignment}
\pysrcexcept{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent} This is because strings are \index{immutable}\textbf{immutable} — you can't change a
string once you have created it.  However, lists are \index{mutable}\textbf{mutable},
and their contents can be modified at any time.  As a result, lists
support a number of operations, or \index{methods}\textbf{methods}, that modify the
original value rather than returning a new value.
A method is a function that is associated with a particular object.
A method is called on the object by giving the object's name,
then a period, then the name of the method, and finally the parentheses
containing any arguments.  For example, in the following code we use
the \texttt{sort{\linebreak[0]}({\linebreak[0]})} and \texttt{reverse{\linebreak[0]}({\linebreak[0]})} methods:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }chomsky.sort()
\pysrcprompt{>{}>{}> }chomsky.reverse()
\pysrcprompt{>{}>{}> }chomsky
\pysrcoutput{{[}'sleep', 'ideas', 'green', 'furiously', 'colorless'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

As you will see, the prompt reappears immediately on the line after
\texttt{chomsky{\linebreak[0]}.{\linebreak[0]}sort{\linebreak[0]}({\linebreak[0]})} and \texttt{chomsky{\linebreak[0]}.{\linebreak[0]}reverse{\linebreak[0]}({\linebreak[0]})}. That is because these
methods do not produce a new list, but instead modify the original list
stored in the variable \texttt{chomsky}.

Lists also have an \texttt{append{\linebreak[0]}({\linebreak[0]})} method for adding items to the
end of the list and an \texttt{index{\linebreak[0]}({\linebreak[0]})} method for finding the index
of particular items in the list:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }chomsky.append(\pysrcstring{'said'})
\pysrcprompt{>{}>{}> }chomsky.append(\pysrcstring{'Chomsky'})
\pysrcprompt{>{}>{}> }chomsky
\pysrcoutput{{[}'sleep', 'ideas', 'green', 'furiously', 'colorless', 'said', 'Chomsky'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }chomsky.index(\pysrcstring{'green'})
\pysrcoutput{2}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Finally, just as a reminder, you can create lists of any values you like.
As you can see in the following example for a lexical entry, the values
in a list do not even have to have the same type (though this is usually
not a good idea, as we will explain in \href{\#sec-back-to-the-basics}{Section 5.2}).
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }bat = {[}\pysrcstring{'bat'}, {[}{[}1, \pysrcstring{'n'}, \pysrcstring{'flying mammal'}{]}, {[}2, \pysrcstring{'n'}, \pysrcstring{'striking instrument'}{]}{]}{]}
\pysrcprompt{>{}>{}>}}\end{alltt}


%___________________________________________________________________________

\subsection{Working on Sequences One Item at a Time}
\label{working-on-sequences-one-item-at-a-time}

We have shown you how to create lists, and how to index and  manipulate them in
various ways. Often it is useful to step through a list and process
each item in some way.  We do this using a \texttt{\pysrckeyword{for}} loop.
This is our first example of a \index{control structure}\textbf{control structure} in Python, a statement
that \docutilsroleemphasis{controls} how other statements are run:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} num \pysrckeyword{in} {[}1, 2, 3{]}:
\pysrcmore{... }    \pysrckeyword{print} \pysrcstring{'The number is'}, num
\pysrcmore{...}
\pysrcoutput{The number is 1}
\pysrcoutput{The number is 2}
\pysrcoutput{The number is 3}}\end{alltt}

The \texttt{\pysrckeyword{for}} loop has the general form: \texttt{\pysrckeyword{for}} \emph{variable} \texttt{\pysrckeyword{in}} \emph{sequence}
followed by a colon, then an indented block of code.
The first time through the
loop, the variable is assigned to the first item in the sequence,
i.e. \texttt{num} has the value \texttt{1}.
This program runs the statement \texttt{\pysrckeyword{print} \pysrcstring{'{\linebreak[0]}The{\linebreak[0]} {\linebreak[0]}number{\linebreak[0]} {\linebreak[0]}is{\linebreak[0]}'},{\linebreak[0]} {\linebreak[0]}num} for this
value of \texttt{num}, before returning to the top of the loop and
assigning the second item to the variable.  Once all items in the
sequence have been processed, the loop finishes.

Now let's try the same idea with a list of words:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }chomsky = {[}\pysrcstring{'colorless'}, \pysrcstring{'green'}, \pysrcstring{'ideas'}, \pysrcstring{'sleep'}, \pysrcstring{'furiously'}{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} chomsky:
\pysrcmore{... }    \pysrckeyword{print} len(word), word{[}-1{]}, word
\pysrcmore{...}
\pysrcoutput{9 s colorless}
\pysrcoutput{5 n green}
\pysrcoutput{5 s ideas}
\pysrcoutput{5 p sleep}
\pysrcoutput{9 y furiously}}\end{alltt}

The first time through this loop, the variable is assigned the value \texttt{\pysrcstring{'{\linebreak[0]}colorless{\linebreak[0]}'}}.
This program runs the statement \texttt{\pysrckeyword{print} {\linebreak[0]}len{\linebreak[0]}({\linebreak[0]}word{\linebreak[0]}){\linebreak[0]},{\linebreak[0]} {\linebreak[0]}word{\linebreak[0]}{[}{\linebreak[0]}-{\linebreak[0]}1{\linebreak[0]}{]}{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}word} for this
value, to produce the output line: \texttt{9{\linebreak[0]} {\linebreak[0]}s{\linebreak[0]} {\linebreak[0]}colorless}.
This process is known as \index{iteration}\textbf{iteration}.
Each iteration of the \texttt{\pysrckeyword{for}} loop starts by assigning the next
item of the list \texttt{chomsky} to the \index{loop variable}\textbf{loop variable} \texttt{word}. Then
the indented \index{body}\textbf{body} of the loop is run.  Here the body consists of
a single command, but in general the body can contain as many lines of
code as you want, so long as they are all indented by the same amount.
(We recommend that you always use exactly 4 spaces for indentation, and that
you never use tabs.)

We can run another \texttt{\pysrckeyword{for}} loop over the Chomsky nonsense sentence,
and calculate the average word length.  As you will see, this program uses
the \texttt{len{\linebreak[0]}({\linebreak[0]})} function in two ways: to count the number of characters in
a word, and to count the number of words in a phrase.  Note that \texttt{x{\linebreak[0]} {\linebreak[0]}+{\linebreak[0]}={\linebreak[0]} {\linebreak[0]}y}
is shorthand for \texttt{x{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}x{\linebreak[0]} {\linebreak[0]}+{\linebreak[0]} {\linebreak[0]}y}; this idiom allows us to
\index{increment}\textbf{increment} the \texttt{total} variable each time the loop is run.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }total = 0
\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} chomsky:
\pysrcmore{... }    total += len(word)
\pysrcmore{...}
\pysrcprompt{>{}>{}> }total / len(chomsky)
\pysrcoutput{6}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

We can also write \texttt{\pysrckeyword{for}} loops to iterate over the characters
in strings.  This \texttt{\pysrckeyword{print}} statement ends with a trailing comma, which is
how we tell Python not to print a newline at the end.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = \pysrcstring{'colorless green ideas sleep furiously'}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} char \pysrckeyword{in} sent:
\pysrcmore{... }    \pysrckeyword{print} char,
\pysrcmore{...}
\pysrcoutput{c o l o r l e s s   g r e e n   i d e a s   s l e e p   f u r i o u s l y}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

A note of caution: we have now iterated over words and characters,
using expressions like \texttt{\pysrckeyword{for} {\linebreak[0]}word{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}sent{\linebreak[0]}:} and \texttt{\pysrckeyword{for} {\linebreak[0]}char{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}sent{\linebreak[0]}:}.
Remember that, to Python, \texttt{word} and \texttt{char} are meaningless variable names,
and we could just as well have written \texttt{\pysrckeyword{for} {\linebreak[0]}foo123{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}sent{\linebreak[0]}:}.
The interpreter simply iterates over the items in the sequence,
quite oblivious to what kind of object they represent, e.g.:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} foo123 \pysrckeyword{in} \pysrcstring{'colorless green ideas sleep furiously'}:
\pysrcmore{... }    \pysrckeyword{print} foo123,
\pysrcmore{...}
\pysrcoutput{c o l o r l e s s   g r e e n   i d e a s   s l e e p   f u r i o u s l y}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{for} foo123 \pysrckeyword{in} {[}\pysrcstring{'colorless'}, \pysrcstring{'green'}, \pysrcstring{'ideas'}, \pysrcstring{'sleep'}, \pysrcstring{'furiously'}{]}:
\pysrcmore{... }    \pysrckeyword{print} foo123,
\pysrcmore{...}
\pysrcoutput{colorless green ideas sleep furiously}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

However, you should try to choose 'sensible' names for loop variables because
it will make your code more readable.


%___________________________________________________________________________

\subsection{String Formatting}
\label{string-formatting}

The output of a program is usually structured to make the information
easily digestible by a reader.  Instead of running some code and then
manually inspecting the contents of a variable, we would like the code
to tabulate some output.  We already saw this above in the first
\texttt{\pysrckeyword{for}} loop example that used a list of words,
where each line of output was similar to \texttt{5{\linebreak[0]} {\linebreak[0]}p{\linebreak[0]} {\linebreak[0]}sleep},
consisting of a word length, the last character of the word,
then the word itself.

There are many ways we might want to format such output.  For
instance, we might want to place the length value in parentheses \docutilsroleemphasis{after} the
word, and print all the output on a single line:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} chomsky:
\pysrcmore{... }    \pysrckeyword{print} word, \pysrcstring{'('}, len(word), \pysrcstring{'),'},
\pysrcoutput{colorless ( 9 ), green ( 5 ), ideas ( 5 ), sleep ( 5 ), furiously ( 9 ),}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

However, this approach has a couple of problems.  First, the \texttt{\pysrckeyword{print}}
statement intermingles variables and punctuation, making it a little
difficult to read.  Second, the output has spaces around every item
that was printed.  A cleaner way to produce structured output uses
Python's \index{string formatting expressions}\textbf{string formatting expressions}. Before diving into
clever formatting tricks, however, let's look at a really simple example. We
are going to use a special symbol, \texttt{{\%}{\linebreak[0]}s}, as a placeholder in
strings. Once we have a string containing this placeholder, we follow
it with a single \texttt{{\%}}
and then a value \texttt{v}. Python then returns a new string where
\texttt{v} has been slotted in to replace \texttt{{\%}{\linebreak[0]}s}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrcstring{"I want a {\%}s right now"} {\%} \pysrcstring{"coffee"}
\pysrcoutput{'I want a coffee right now'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

In fact, we can have a number of placeholders, but following the \texttt{{\%}} operator
we need to specify exactly the same number of values.  Note that the
parentheses are required.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrcstring{"{\%}s wants a {\%}s {\%}s"} {\%} (\pysrcstring{"Lee"}, \pysrcstring{"sandwich"}, \pysrcstring{"for lunch"})
\pysrcoutput{'Lee wants a sandwich for lunch'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

We can also provide the values for the placeholders indirectly. Here's
an example using a \texttt{\pysrckeyword{for}} loop:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }menu = {[}\pysrcstring{'sandwich'}, \pysrcstring{'spam fritter'}, \pysrcstring{'pancake'}{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} snack \pysrckeyword{in} menu:
\pysrcmore{... }    \pysrcstring{"Lee wants a {\%}s right now"} {\%} snack
\pysrcmore{...}
\pysrcoutput{'Lee wants a sandwich right now'}
\pysrcoutput{'Lee wants a spam fritter right now'}
\pysrcoutput{'Lee wants a pancake right now'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

We oversimplified things when we said that placeholders were
of the form \texttt{{\%}{\linebreak[0]}s}; in fact, this is a complex object, called a
\index{conversion specifier}\textbf{conversion specifier}. This has to start with the \texttt{{\%}}
character, and ends with conversion character such as \texttt{s{\linebreak[0]}`{\linebreak[0]} \pysrckeyword{or} {\linebreak[0]}`{\linebreak[0]}`{\linebreak[0]}d}. The \texttt{{\%}{\linebreak[0]}s}
specifier tells Python that the corresponding variable is a string (or
should be converted into a string), while the \texttt{{\%}{\linebreak[0]}d} specifier
indicates that the corresponding variable should be converted into a
decimal representation. The string containing conversion specifiers is
called a \index{format string}\textbf{format string}.

Picking up on the \texttt{\pysrckeyword{print}} example that we opened this section with,
here's how we can use two different kinds of conversion specifier:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} chomsky:
\pysrcmore{... }    \pysrckeyword{print} \pysrcstring{"{\%}s ({\%}d),"} {\%} (word, len(word)),
\pysrcoutput{colorless (9), green (5), ideas (5), sleep (5), furiously (9),}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

To summarize, string formatting is accomplished with a three-part
object having the syntax: \docutilsroleplaceholder{format} \texttt{{\%}} \docutilsroleplaceholder{values}.  The
\docutilsroleplaceholder{format} section is a string containing format specifiers
such as \texttt{{\%}{\linebreak[0]}s} and \texttt{{\%}{\linebreak[0]}d} that Python will replace with the supplied
values.  The \docutilsroleplaceholder{values} section of a formatting string is a parenthesized
list containing exactly as many items as there are format specifiers in the
\docutilsroleplaceholder{format} section.  In the case that there is just one item, the
parentheses can be left out.  (We will discuss Python's
string-formatting expressions in more detail in \href{\#sec-lining-things-up}{Section 5.3.2}).

In the above example, we used a trailing comma to suppress the
printing of a newline. Suppose, on the other hand, that we want to
introduce some additional newlines in our output. We can accomplish
this by inserting the ``special'' character \texttt{{\textbackslash}{\linebreak[0]}n} into the \texttt{\pysrckeyword{print}} string:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} chomsky:
\pysrcmore{... }   \pysrckeyword{print} \pysrcstring{"Word = {\%}s{\textbackslash}nIndex = {\%}s{\textbackslash}n*****"} {\%} (word, chomsky.index(word))
\pysrcmore{...}
\pysrcoutput{Word = colorless}
\pysrcoutput{Index = 0}
\pysrcoutput{*****}
\pysrcoutput{Word = green}
\pysrcoutput{Index = 1}
\pysrcoutput{*****}
\pysrcoutput{Word = ideas}
\pysrcoutput{Index = 2}
\pysrcoutput{*****}
\pysrcoutput{Word = sleep}
\pysrcoutput{Index = 3}
\pysrcoutput{*****}
\pysrcoutput{Word = furiously}
\pysrcoutput{Index = 4}
\pysrcoutput{*****}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}


%___________________________________________________________________________

\subsection{Converting Between Strings and Lists}
\label{converting-between-strings-and-lists}

Often we want to convert between a string containing a space-separated
list of words and a list of strings. Let's first consider turning a
list into a string. One way of doing this is as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }s = \pysrcstring{'{}'}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} chomsky:
\pysrcmore{... }   s += \pysrcstring{' '} + word
\pysrcmore{...}
\pysrcprompt{>{}>{}> }s
\pysrcoutput{' colorless green ideas sleep furiously'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

One drawback of this approach is that we have an unwanted space at the
start of \texttt{s}.  It is more convenient to use the \texttt{join{\linebreak[0]}({\linebreak[0]})} method.
We specify the string to be used as the ``glue'', followed by a
period, followed by the \texttt{join{\linebreak[0]}({\linebreak[0]})} function.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = \pysrcstring{' '}.join(chomsky)
\pysrcprompt{>{}>{}> }sent
\pysrcoutput{'colorless green ideas sleep furiously'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

So \texttt{\pysrcstring{'{\linebreak[0]} {\linebreak[0]}'}.{\linebreak[0]}join{\linebreak[0]}({\linebreak[0]}chomsky{\linebreak[0]})} means: take all the items in \texttt{chomsky} and
concatenate them as one big string, using \texttt{\pysrcstring{'{\linebreak[0]} {\linebreak[0]}'}} as a spacer between
the items.

Now let's try to reverse the process: that is, we want to convert a
string into a list. Again, we could start off with an empty list
\texttt{{[}{\linebreak[0]}{]}} and \texttt{append{\linebreak[0]}({\linebreak[0]})} to it within a \texttt{\pysrckeyword{for}} loop.  But as before,
there is a more succinct way of achieving the same goal.  This time,
we will \docutilsroleemphasis{split} the new string \texttt{sent} on whitespace:
% why did we have this?  >>> sent.split(' ')
% 
% >>> sent.split()
% ['colorless', 'green', 'ideas', 'sleep', 'furiously']
% >>> 

To consolidate your understanding of joining and splitting strings,
let's try the same thing using a semicolon as the separator:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = \pysrcstring{';'}.join(chomsky)
\pysrcprompt{>{}>{}> }sent
\pysrcoutput{'colorless;green;ideas;sleep;furiously'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }sent.split(\pysrcstring{';'})
\pysrcoutput{{[}'colorless', 'green', 'ideas', 'sleep', 'furiously'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

To be honest, many people find the notation for \texttt{join{\linebreak[0]}({\linebreak[0]})} rather
unintuitive.
There is another function for converting lists to strings, again
called \texttt{join{\linebreak[0]}({\linebreak[0]})} which is called directly on the list. It uses whitespace by
default as the ``glue''. However, we need to explicitly \index{import}\textbf{import} this
function into our code. One way of doing this is as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} string
\pysrcprompt{>{}>{}> }string.join(chomsky)
\pysrcoutput{'colorless green ideas sleep furiously'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Here, we imported something called \texttt{string}, and then called the
function \texttt{string{\linebreak[0]}.{\linebreak[0]}join{\linebreak[0]}({\linebreak[0]})}. In passing, if we want to use something
other than whitespace as ``glue'', we just specify this as a second
parameter:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }string.join(chomsky, \pysrcstring{';'})
\pysrcoutput{'colorless;green;ideas;sleep;furiously'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

We will see other examples of statements with \texttt{\pysrckeyword{import}} later in this
chapter.  In general, we use \texttt{\pysrckeyword{import}} statements when we want to get
access to Python code that doesn't already come as part of core
Python. This code will exist somewhere as one or more files. Each such
file corresponds to a Python \index{module}\textbf{module} — this is a way of
grouping together code and data that we regard as reusable. When you
write down some Python statements in a file, you are in effect
creating a new Python module. And you can make your code depend on
another module by using the \texttt{\pysrckeyword{import}} statement. In our example
earlier, we imported the module \texttt{string} and then used the
\texttt{join{\linebreak[0]}({\linebreak[0]})} function from that module. By adding \texttt{string{\linebreak[0]}.} to the
beginning of \texttt{join{\linebreak[0]}({\linebreak[0]})}, we make it clear to the Python interpreter
that the definition of \texttt{join{\linebreak[0]}({\linebreak[0]})} is given in the \texttt{string}
module. An alternative, and equally valid, approach is to use the
\texttt{\pysrckeyword{from}} \emph{module} \texttt{\pysrckeyword{import}} \emph{identifier} statement, as shown in the
next example:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} string \pysrckeyword{import} join
\pysrcprompt{>{}>{}> }join(chomsky)
\pysrcoutput{'colorless green ideas sleep furiously'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

In this case, the name \texttt{join} is added to all the other identifier that we
have defined in the body of our programme, and we can use it to call a
function like any other.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

If you are creating a file to contain some of your Python
code, do \emph{not} name your file \texttt{nltk{\linebreak[0]}.{\linebreak[0]}py}: it may get imported in
place of the ``real'' NLTK package. (When it imports modules, Python
first looks in the current folder / directory.)
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\subsection{Mini-Review}
\label{mini-review}

Strings and lists are both kind of \index{sequence}\textbf{sequence}. As such, they can
both be indexed and sliced:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }query = \pysrcstring{'Who knows?'}
\pysrcprompt{>{}>{}> }beatles = {[}\pysrcstring{'john'}, \pysrcstring{'paul'}, \pysrcstring{'george'}, \pysrcstring{'ringo'}{]}
\pysrcprompt{>{}>{}> }query{[}2{]}
\pysrcoutput{'o'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }beatles{[}2{]}
\pysrcoutput{'george'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }query{[}:2{]}
\pysrcoutput{'Wh'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }beatles{[}:2{]}
\pysrcoutput{{[}'john', 'paul'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Similarly, strings can be concatenated and so can lists (though not
with each other!):
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }newstring = query + \pysrcstring{" I don't"}
\pysrcprompt{>{}>{}> }newlist = beatles + {[}\pysrcstring{'brian'}, \pysrcstring{'george'}{]}}\end{alltt}

What's the difference between strings and lists as far as NLP is
concerned? As we will see in \href{\#chap-words}{Chapter 2}, when we open a file
for reading into a Python program, what we get initially is a string,
corresponding to the contents of the whole file. If we try to use a \texttt{\pysrckeyword{for}} loop to
process the elements of this string, all we can pick out are the
individual characters in the string — we don't get to choose the
granularity. By contrast, the elements of a list can be as big or
small as we like: for example, they could be paragraphs, sentence,
phrases, words, characters. So lists have this huge advantage, that we
can be really flexible about the elements they contain, and
correspondingly flexible about what the downstream processing will act
on. So one of the first things we are likely to do in a piece of NLP
code is convert a string into a list (of strings). Conversely, when we
want to write our results to a file, or to a terminal, we will usually
convert them to a string.


%___________________________________________________________________________

\subsection{Exercises}
\label{id2}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Using the Python interactive interpreter, experiment with
the examples in this section.  Think of a sentence and
represent it as a list of strings, e.g. {[}'Hello', 'world'{]}.
Try the various operations for indexing, slicing and sorting the elements
of your list.  Extract individual items (strings), and perform
some of the string operations on them.

\item {} 
☼ Split \texttt{sent} on some other character, such as \texttt{\pysrcstring{'{\linebreak[0]}s{\linebreak[0]}'}}.

\item {} 
☼ We pointed out that when \texttt{phrase} is a list, \texttt{phrase{\linebreak[0]}.{\linebreak[0]}reverse{\linebreak[0]}({\linebreak[0]})}
returns a modified version of \texttt{phrase} rather than a new list. On
the other hand, we can use the slice trick mentioned in the
exercises for the previous section, \texttt{{[}{\linebreak[0]}:{\linebreak[0]}:{\linebreak[0]}-{\linebreak[0]}1{\linebreak[0]}{]}} to create a \docutilsroleemphasis{new} reversed list
without changing \texttt{phrase}. Show how you can confirm this
difference in behavior.

\item {} 
☼ We have seen how to represent a sentence as a list of words, where
each word is a sequence of characters.  What does \texttt{phrase1{\linebreak[0]}{[}{\linebreak[0]}2{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}2{\linebreak[0]}{]}} do?
Why?  Experiment with other index values.

\item {} 
☼ Write a \texttt{\pysrckeyword{for}} loop to print out the characters of a string, one per line.

\item {} 
☼ What is the difference between calling \texttt{split} on a string
with no argument or with \texttt{\pysrcstring{'{\linebreak[0]} {\linebreak[0]}'}} as the argument,
e.g. \texttt{sent{\linebreak[0]}.{\linebreak[0]}split{\linebreak[0]}({\linebreak[0]})} versus \texttt{sent{\linebreak[0]}.{\linebreak[0]}split{\linebreak[0]}(\pysrcstring{'{\linebreak[0]} {\linebreak[0]}'})}?  What happens
when the string being split contains tab characters, consecutive
space characters, or a sequence of tabs and spaces?  (In IDLE you
will need to use \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}t{\linebreak[0]}'}} to enter a tab character.)

\item {} 
☼ Create a variable \texttt{words} containing a list of words.
Experiment with \texttt{words{\linebreak[0]}.{\linebreak[0]}sort{\linebreak[0]}({\linebreak[0]})} and \texttt{sorted{\linebreak[0]}({\linebreak[0]}words{\linebreak[0]})}.
What is the difference?

\item {} 
☼ Earlier, we asked you to use a text editor to create a file
called \texttt{test{\linebreak[0]}.{\linebreak[0]}py}, containing the single line \texttt{msg{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} \pysrcstring{'{\linebreak[0]}Hello}
\pysrcstring{World{\linebreak[0]}'}}. If you haven't already done this (or can't find the file),
go ahead and do it now. Next, start up a new session with the
Python interpreter, and enter the expression \texttt{msg} at the prompt.
You will get an error from the interpreter. Now, try the following
(note that you have to leave off the \texttt{.{\linebreak[0]}py} part of the filename):
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} test \pysrckeyword{import} msg
\pysrcprompt{>{}>{}> }msg}\end{alltt}

This time, Python should return with a value. You can also try
\texttt{\pysrckeyword{import} {\linebreak[0]}test}, in which case Python should be able to
evaluate the expression \texttt{test{\linebreak[0]}.{\linebreak[0]}msg} at the prompt.

\item {} 
◑ Process the list \texttt{chomsky} using a \texttt{\pysrckeyword{for}} loop, and store the
result in a new list \texttt{lengths}.  Hint: begin by assigning the
empty list to \texttt{lengths}, using \texttt{lengths{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}{]}}. Then each time
through the loop, use \texttt{append{\linebreak[0]}({\linebreak[0]})} to add another length value to
the list.

\item {} 
◑ Define a variable \texttt{silly} to contain the string:
\texttt{\pysrcstring{'{\linebreak[0]}newly{\linebreak[0]} {\linebreak[0]}formed{\linebreak[0]} {\linebreak[0]}bland{\linebreak[0]} {\linebreak[0]}ideas{\linebreak[0]} {\linebreak[0]}are{\linebreak[0]} {\linebreak[0]}inexpressible{\linebreak[0]} {\linebreak[0]}in{\linebreak[0]} {\linebreak[0]}an{\linebreak[0]} {\linebreak[0]}infuriating}
\pysrcstring{way{\linebreak[0]}'}}.  (This happens to be the legitimate interpretation that
bilingual English-Spanish speakers can assign to Chomsky's
famous phrase, according to Wikipedia).  Now write code to perform
the following tasks:
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Split \texttt{silly} into a list of strings, one per
word, using Python's \texttt{split{\linebreak[0]}({\linebreak[0]})} operation, and save
this to a variable called \texttt{bland}.

\item {} 
Extract the second letter of each word in \texttt{silly} and join
them into a string, to get \texttt{\pysrcstring{'{\linebreak[0]}eoldrnnnna{\linebreak[0]}'}}.

\item {} 
Combine the words in \texttt{bland} back into a single string, using \texttt{join{\linebreak[0]}({\linebreak[0]})}.
Make sure the words in the resulting string are separated with
whitespace.

\item {} 
Print the words of \texttt{silly} in alphabetical order, one per line.

\end{list}

\item {} 
◑ The \texttt{index{\linebreak[0]}({\linebreak[0]})} function can be used to look up items in sequences.
For example, \texttt{\pysrcstring{'{\linebreak[0]}inexpressible{\linebreak[0]}'}.{\linebreak[0]}index{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}e{\linebreak[0]}'})} tells us the index of the
first position of the letter \texttt{e}.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
What happens when you look up a substring, e.g. \texttt{\pysrcstring{'{\linebreak[0]}inexpressible{\linebreak[0]}'}.{\linebreak[0]}index{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}re{\linebreak[0]}'})}?

\item {} 
Define a variable \texttt{words} containing a list of words.  Now use \texttt{words{\linebreak[0]}.{\linebreak[0]}index{\linebreak[0]}({\linebreak[0]})}
to look up the position of an individual word.

\item {} 
Define a variable \texttt{silly} as in the exercise above.
Use the \texttt{index{\linebreak[0]}({\linebreak[0]})} function in combination with list slicing to
build a list \texttt{phrase} consisting of all the words up to (but not
including) \texttt{\pysrckeyword{in}} in \texttt{silly}.

\end{list}

\end{list}


%___________________________________________________________________________

\section{Making Decisions}
\label{making-decisions}

So far, our simple programs have been able to manipulate sequences of
words, and perform some operation on each one.  We applied this to lists
consisting of a few words, but the approach works the same for lists of
arbitrary size, containing thousands of items.  Thus, such programs
have some interesting qualities: (i) the ability to work with
language, and (ii) the potential to save human effort through
automation.  Another useful feature of programs is their ability to
\docutilsroleemphasis{make decisions} on our behalf; this is our focus in this section.


%___________________________________________________________________________

\subsection{Making Simple Decisions}
\label{making-simple-decisions}

Most programming languages permit us to execute a block of code when a
\index{conditional expression}\textbf{conditional expression}, or \texttt{\pysrckeyword{if}} statement, is satisfied.  In
the following program, we have created a variable called \texttt{word}
containing the string value \texttt{\pysrcstring{'{\linebreak[0]}cat{\linebreak[0]}'}}. The \texttt{\pysrckeyword{if}} statement then
checks whether the condition \texttt{len{\linebreak[0]}({\linebreak[0]}word{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}<{\linebreak[0]} {\linebreak[0]}5} is true.  Because the
conditional expression is true, the body of the \texttt{\pysrckeyword{if}} statement is
invoked and the \texttt{\pysrckeyword{print}} statement is executed.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }word = \pysrcstring{"cat"}
\pysrcprompt{>{}>{}> }\pysrckeyword{if} len(word) < 5:
\pysrcmore{... }  \pysrckeyword{print} \pysrcstring{'word length is less than 5'}
\pysrcmore{...}
\pysrcoutput{word length is less than 5}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

If we change the conditional expression to \texttt{len{\linebreak[0]}({\linebreak[0]}word{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}>{\linebreak[0]}={\linebreak[0]} {\linebreak[0]}5},
to check that the length of \texttt{word} is greater than or equal to \texttt{5},
then the conditional expression will no longer be true,
and the body of the \texttt{\pysrckeyword{if}} statement will not be run:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{if} len(word) >= 5:
\pysrcmore{... }  \pysrckeyword{print} \pysrcstring{'word length is greater than or equal to 5'}
\pysrcmore{...}
\pysrcprompt{>{}>{}>}}\end{alltt}

The \texttt{\pysrckeyword{if}} statement, just like the \texttt{\pysrckeyword{for}} statement above is a
\index{control structure}\textbf{control structure}.  An \texttt{\pysrckeyword{if}} statement is a control structure
because it controls whether the code in the body will be run.  You
will notice that both \texttt{\pysrckeyword{if}} and \texttt{\pysrckeyword{for}} have a colon at the end of
the line, before the indentation begins. That's because all Python
control structures end with a colon.

What if we want to do something when the conditional expression is not true?
The answer is to add an \texttt{\pysrckeyword{else}} clause to the \texttt{\pysrckeyword{if}} statement:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{if} len(word) >= 5:
\pysrcmore{... }  \pysrckeyword{print} \pysrcstring{'word length is greater than or equal to 5'}
\pysrcmore{... }\pysrckeyword{else}:
\pysrcmore{... }  \pysrckeyword{print} \pysrcstring{'word length is less than 5'}
\pysrcmore{...}
\pysrcoutput{word length is less than 5}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Finally, if we want to test multiple conditions in one go, we can use an
\texttt{\pysrckeyword{elif}} clause that acts like an \texttt{\pysrckeyword{else}} and an \texttt{\pysrckeyword{if}} combined:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{if} len(word) < 3:
\pysrcmore{... }  \pysrckeyword{print} \pysrcstring{'word length is less than three'}
\pysrcmore{... }\pysrckeyword{elif} len(word) == 3:
\pysrcmore{... }  \pysrckeyword{print} \pysrcstring{'word length is equal to three'}
\pysrcmore{... }\pysrckeyword{else}:
\pysrcmore{... }  \pysrckeyword{print} \pysrcstring{'word length is greater than three'}
\pysrcmore{...}
\pysrcoutput{word length is equal to three}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

It's worth noting that in the condition part of an \texttt{\pysrckeyword{if}} statement, a
nonempty string or list is evaluated as true, while an empty string or
list evaluates as false.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }mixed = {[}\pysrcstring{'cat'}, \pysrcstring{'{}'}, {[}\pysrcstring{'dog'}{]}, {[}{]}{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} element \pysrckeyword{in} mixed:
\pysrcmore{... }    \pysrckeyword{if} element:
\pysrcmore{... }        \pysrckeyword{print} element
\pysrcmore{...}
\pysrcoutput{cat}
\pysrcoutput{{[}'dog'{]}}}\end{alltt}

That is, we \emph{don't} need to say \texttt{\pysrckeyword{if} {\linebreak[0]}len{\linebreak[0]}({\linebreak[0]}element{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}>{\linebreak[0]} {\linebreak[0]}0{\linebreak[0]}:} in the
condition.

What's the difference between using \texttt{\pysrckeyword{if}.{\linebreak[0]}.{\linebreak[0]}.\pysrckeyword{elif}} as opposed to using
a couple of \texttt{\pysrckeyword{if}} statements in a row? Well, consider the following
situation:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }animals = {[}\pysrcstring{'cat'}, \pysrcstring{'dog'}{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{if} \pysrcstring{'cat'} \pysrckeyword{in} animals:
\pysrcmore{... }    \pysrckeyword{print} 1
\pysrcmore{... }\pysrckeyword{elif} \pysrcstring{'dog'} \pysrckeyword{in} animals:
\pysrcmore{... }    \pysrckeyword{print} 2
\pysrcmore{...}
\pysrcoutput{1}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Since the \texttt{\pysrckeyword{if}} clause of the statement is satisfied, Python never
tries to evaluate the \texttt{\pysrckeyword{elif}} clause, so we never get to print out
\texttt{2}. By contrast, if we replaced the \texttt{\pysrckeyword{elif}} by an \texttt{\pysrckeyword{if}}, then we
would print out both \texttt{1} and \texttt{2}. So an \texttt{\pysrckeyword{elif}} clause
potentially gives us more information than a bare \texttt{\pysrckeyword{if}} clause; when
it evaluates to true, it tells us not only that the condition is
satisfied, but also that the condition of the main \texttt{\pysrckeyword{if}} clause was
\emph{not} satisfied.


%___________________________________________________________________________

\subsection{Conditional Expressions}
\label{conditional-expressions}

Python supports a wide range of operators like \texttt{<} and \texttt{>{\linebreak[0]}=} for
testing the relationship between values. The full set of these \index{relational
operators}\textbf{relational
operators} are shown in \href{\#inequalities}{Table 1.1}.
\hypertarget{inequalities}{}\begin{quote}

\leavevmode
\begin{longtable}[c]{|p{0.110\locallinewidth}|p{0.551\locallinewidth}|}
\hline
\textbf{
Operator
} & \textbf{
Relationship
} \\
\hline
\endhead

\texttt{<}
 & 
less than
 \\
\hline

\texttt{<{\linebreak[0]}=}
 & 
less than or equal to
 \\
\hline

\texttt{={\linebreak[0]}=}
 & 
equal to (note this is two not one \texttt{=} sign)
 \\
\hline

\texttt{!{\linebreak[0]}=}
 & 
not equal to
 \\
\hline

\texttt{>}
 & 
greater than
 \\
\hline

\texttt{>{\linebreak[0]}=}
 & 
greater than or equal to
 \\
\hline
\caption{}\end{longtable}

Conditional Expressions
\end{quote}

Normally we use conditional expressions as part of an \texttt{\pysrckeyword{if}} statement.
However, we can test these relational operators directly at the prompt:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }3 < 5
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }5 < 3
\pysrcoutput{False}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{not} 5 < 3
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Here we see that these expressions have \index{Boolean}\textbf{Boolean} values, namely
\texttt{True} or \texttt{False}. \texttt{\pysrckeyword{not}} is a Boolean operator, and flips the
truth value of Boolean statement.

Strings and lists also support conditional operators:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }word = \pysrcstring{'sovereignty'}
\pysrcprompt{>{}>{}> }\pysrcstring{'sovereign'} \pysrckeyword{in} word
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'gnt'} \pysrckeyword{in} word
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'pre'} \pysrckeyword{not} \pysrckeyword{in} word
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'Hello'} \pysrckeyword{in} {[}\pysrcstring{'Hello'}, \pysrcstring{'World'}{]}
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'Hell'} \pysrckeyword{in} {[}\pysrcstring{'Hello'}, \pysrcstring{'World'}{]}
\pysrcoutput{False}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Strings also have methods for testing what appears at the
beginning and the end of a string (as opposed to just anywhere in the
string:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }word.startswith(\pysrcstring{'sovereign'})
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }word.endswith(\pysrcstring{'ty'})
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}


%___________________________________________________________________________

\subsection{Iteration, Items, and \texttt{\pysrckeyword{if}}}
\label{iteration-items-and-if}

Now it is time to put some of the pieces together. We are going to take
the string \texttt{\pysrcstring{'{\linebreak[0]}how{\linebreak[0]} {\linebreak[0]}now{\linebreak[0]} {\linebreak[0]}brown{\linebreak[0]} {\linebreak[0]}cow{\linebreak[0]}'}} and print out all of the words ending
in \texttt{\pysrcstring{'{\linebreak[0]}ow{\linebreak[0]}'}}.  Let's build the program up in stages.
The first step is to split the string into a list of words:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sentence = \pysrcstring{'how now brown cow'}
\pysrcprompt{>{}>{}> }words = sentence.split()
\pysrcprompt{>{}>{}> }words
\pysrcoutput{{[}'how', 'now', 'brown', 'cow'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Next, we need to iterate over the words in the list. Just so we don't
get ahead of ourselves, let's print each word, one per line:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} words:
\pysrcmore{... }    \pysrckeyword{print} word
\pysrcmore{...}
\pysrcoutput{how}
\pysrcoutput{now}
\pysrcoutput{brown}
\pysrcoutput{cow}}\end{alltt}

The next stage is to only print out the words if they end in the
string \texttt{\pysrcstring{'{\linebreak[0]}ow{\linebreak[0]}'}}.  Let's check that we know how to do this first:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrcstring{'how'}.endswith(\pysrcstring{'ow'})
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'brown'}.endswith(\pysrcstring{'ow'})
\pysrcoutput{False}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Now we are ready to put an \texttt{\pysrckeyword{if}} statement inside the \texttt{\pysrckeyword{for}} loop.
Here is the complete program:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sentence = \pysrcstring{'how now brown cow'}
\pysrcprompt{>{}>{}> }words = sentence.split()
\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} words:
\pysrcmore{... }    \pysrckeyword{if} word.endswith(\pysrcstring{'ow'}):
\pysrcmore{... }        \pysrckeyword{print} word
\pysrcmore{...}
\pysrcoutput{how}
\pysrcoutput{now}
\pysrcoutput{cow}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

As you can see, even with this small amount of Python knowledge it is
possible to develop useful programs. The key idea is to develop the
program in pieces, testing that each one does what you expect, and
then combining them to produce whole programs.  This is why the Python
interactive interpreter is so invaluable, and why you should get
comfortable using it.


%___________________________________________________________________________

\subsection{A Taster of Data Types}
\label{a-taster-of-data-types}

Integers, strings and lists are all kinds of \index{data types}\textbf{data types} in
Python, and have types \texttt{int}, \texttt{str} and \texttt{list} respectively. In
fact, every value in Python has a type. Python's \texttt{type{\linebreak[0]}({\linebreak[0]})} function
will tell you what an object's type is:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }oddments = {[}\pysrcstring{'cat'}, \pysrcstring{'cat'}.index(\pysrcstring{'a'}), \pysrcstring{'cat'}.split(){]}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} e \pysrckeyword{in} oddments:
\pysrcmore{... }    type(e)
\pysrcmore{...}
\pysrcoutput{<type 'str'>}
\pysrcoutput{<type 'int'>}
\pysrcoutput{<type 'list'>}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

The type determines what operations you can perform on the data
value. So, for example, we have seen that we can index strings and
lists, but we can't index integers:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }one = \pysrcstring{'cat'}
\pysrcprompt{>{}>{}> }one{[}0{]}
\pysrcoutput{'c'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }two = {[}1, 2, 3{]}
\pysrcprompt{>{}>{}> }two{[}1{]}
\pysrcoutput{2}
\pysrcoutput{}\pysrcprompt{>{}>{}> }three = 1234
\pysrcprompt{>{}>{}> }three{[}2{]}
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{  File "<pyshell{\#}95>", line 1, in -toplevel-}
\pysrcexcept{    three{[}2{]}}
\pysrcexcept{TypeError: 'int' object is unsubscriptable}
\pysrcexcept{}\pysrcprompt{>{}>{}>}}\end{alltt}

The fact that this is a problem with types is signalled by the class of
error, i.e., \texttt{TypeError}; an object being ``unsubscriptable'' means we
can't index into it.

Similarly, we can concatenate strings with strings, and lists with
lists, but we cannot concatenate strings with lists:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }query = \pysrcstring{'Who knows?'}
\pysrcprompt{>{}>{}> }beatles = {[}\pysrcstring{'john'}, \pysrcstring{'paul'}, \pysrcstring{'george'}, \pysrcstring{'ringo'}{]}
\pysrcprompt{>{}>{}> }query + beatles
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{  File "<stdin>", line 1, in <module>}
\pysrcexcept{TypeError: cannot concatenate 'str' and 'list' objects}}\end{alltt}

You may also have noticed that our analogy between operations on
strings and numbers at the beginning of this chapter broke down pretty
soon:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrcstring{'Hi'} * 3
\pysrcoutput{'HiHiHi'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'Hi'} - \pysrcstring{'i'}
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{  File "<stdin>", line 1, in <module>}
\pysrcexcept{TypeError: unsupported operand type(s) for -: 'str' and 'str'}
\pysrcexcept{}\pysrcprompt{>{}>{}> }6 / 2
\pysrcoutput{3}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'Hi'} / 2
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{  File "<stdin>", line 1, in <module>}
\pysrcexcept{TypeError: unsupported operand type(s) for /: 'str' and 'int'}
\pysrcexcept{}\pysrcprompt{>{}>{}>}}\end{alltt}

These error messages are another example of Python telling us that we
have got our data types in a muddle. In the first case, we are told
that the operation of substraction (i.e., \texttt{-}) cannot apply to
objects of type \texttt{str}, while in the second, we are told that
division cannot take \texttt{str} and \texttt{int} as its two operands.


%___________________________________________________________________________

\subsection{Exercises}
\label{id3}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Assign a new value to \texttt{sentence}, namely the string
\texttt{\pysrcstring{'{\linebreak[0]}she{\linebreak[0]} {\linebreak[0]}sells{\linebreak[0]} {\linebreak[0]}sea{\linebreak[0]} {\linebreak[0]}shells{\linebreak[0]} {\linebreak[0]}by{\linebreak[0]} {\linebreak[0]}the{\linebreak[0]} {\linebreak[0]}sea{\linebreak[0]} {\linebreak[0]}shore{\linebreak[0]}'}}, then
write code to perform the following tasks:
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Print all words beginning with \texttt{\pysrcstring{'{\linebreak[0]}sh{\linebreak[0]}'}}:

\item {} 
Print all words longer than 4 characters.

\item {} 
Generate a new sentence that adds the popular
hedge word \texttt{\pysrcstring{'{\linebreak[0]}like{\linebreak[0]}'}} before every word
beginning with \texttt{\pysrcstring{'{\linebreak[0]}se{\linebreak[0]}'}}.  Your result should
be a single string.

\end{list}

\item {} 
☼ Write code to abbreviate text by removing all the vowels.
Define \texttt{sentence} to hold any string you like, then initialize
a new string \texttt{result} to hold the empty string \texttt{\pysrcstring{'{\linebreak[0]}'}}.  Now write
a \texttt{\pysrckeyword{for}} loop to process the string, one character at a time,
and append any non-vowel characters to the result string.

\item {} 
☼ We pointed out that when empty strings and empty lists occur
in the condition part of an \texttt{\pysrckeyword{if}} clause, they evaluate to
false. In this case, they are said to be occuring in a \index{Boolean
context}\textbf{Boolean
context}.
Experiment with different kind of non-Boolean expressions in Boolean
contexts, and see whether they evaluate as true or false.

\item {} 
☼ Review conditional expressions, such as \texttt{\pysrcstring{'{\linebreak[0]}row{\linebreak[0]}'} \pysrckeyword{in} \pysrcstring{'{\linebreak[0]}brown{\linebreak[0]}'}}
and \texttt{\pysrcstring{'{\linebreak[0]}row{\linebreak[0]}'} \pysrckeyword{in} {\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}brown{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}cow{\linebreak[0]}'}{]}}.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Define \texttt{sent} to be the string \texttt{\pysrcstring{'{\linebreak[0]}colorless{\linebreak[0]} {\linebreak[0]}green{\linebreak[0]} {\linebreak[0]}ideas{\linebreak[0]} {\linebreak[0]}sleep{\linebreak[0]} {\linebreak[0]}furiously{\linebreak[0]}'}},
and use conditional expressions to test for the presence of particular words
or substrings.

\item {} 
Now define \texttt{words} to be a list of words contained in the sentence, using
\texttt{sent{\linebreak[0]}.{\linebreak[0]}split{\linebreak[0]}({\linebreak[0]})}, and use conditional expressions to test for the presence
of particular words or substrings.

\end{list}

\item {} 
◑ Write code to convert text into \emph{hAck3r}, where characters are
mapped according to the following table:

\leavevmode
\begin{longtable}[c]{|p{0.121\locallinewidth}|p{0.051\locallinewidth}|p{0.051\locallinewidth}|p{0.051\locallinewidth}|p{0.063\locallinewidth}|p{0.051\locallinewidth}|p{0.110\locallinewidth}|p{0.075\locallinewidth}|}
\hline

Input:
 & 
e
 & 
i
 & 
o
 & 
l
 & 
s
 & 
.
 & 
ate
 \\
\hline

Output:
 & 
3
 & 
1
 & 
0
 & 
{\textbar}
 & 
5
 & 
5w33t!
 & 
8
 \\
\hline
\caption{}\end{longtable}

\end{list}
\hypertarget{getting-organized}{}

%___________________________________________________________________________

\section{Getting Organized}
\label{id4}
\label{getting-organized}

Strings and lists are a simple way to organize data.  In particular,
they \index{map}\textbf{map} from integers to values.  We can ``look up'' a character
in a string using an integer, and we can look up a word in a list of
words using an integer.  These cases are shown in  \href{\#maps01}{Figure 1.5}.
\hypertarget{maps01}{}\begin{figure}[htbp]
\centering

\scalebox{0.128571}{\includegraphics{../images/maps01.png}}
\caption{Sequence Look-up}\end{figure}

However, we need a more flexible way to organize and access our data.
Consider the examples in \href{\#maps02}{Figure 1.6}.
\hypertarget{maps02}{}\begin{figure}[htbp]
\centering

\scalebox{0.128571}{\includegraphics{../images/maps02.png}}
\caption{Dictionary Look-up}\end{figure}

In the case of a phone book, we look up an entry using a \docutilsroleemphasis{name},
and get back a number.  When we type a domain name in a web browser,
the computer looks this up to get back an IP address.  A word
frequency table allows us to look up a word and find its frequency in
a text collection.  In all these cases, we are mapping from names to
numbers, rather than the other way round as with indexing into
sequences.  In general, we would like to be able to map between
arbitrary types of information.  \href{\#linguistic-objects}{Table 1.3} lists a variety
of linguistic objects, along with what they map.
\hypertarget{linguistic-objects}{}\begin{quote}

\leavevmode
\begin{longtable}[c]{|p{0.249\locallinewidth}|p{0.156\locallinewidth}|p{0.435\locallinewidth}|}
\hline
\multirow{2}{0.25\locallinewidth}{\textbf{
Linguistic Object
}} & \multicolumn{2}{l|}{\textbf{
Maps
}} \\
\cline{2-2}
\cline{3-3}
 & \textbf{
from
} & \textbf{
to
} \\
\hline
\endhead

Document Index
 & 
Word
 & 
List of pages (where word is found)
 \\
\hline

Thesaurus
 & 
Word sense
 & 
List of synonyms
 \\
\hline

Dictionary
 & 
Headword
 & 
Entry (part of speech, sense
definitions, etymology)
 \\
\hline

Comparative Wordlist
 & 
Gloss term
 & 
Cognates (list of words, one per
language)
 \\
\hline

Morph Analyzer
 & 
Surface form
 & 
Morphological analysis (list of
component morphemes)
 \\
\hline
\caption{}\end{longtable}

Linguistic Objects as Mappings from Keys to Values
\end{quote}

Most often, we are mapping from a string to some structured object.
For example, a document index maps from a word (which we can represent
as a string), to a list of pages (represented as a list of integers).
In this section, we will see how to represent such mappings in Python.


%___________________________________________________________________________

\subsection{Accessing Data with Data}
\label{accessing-data-with-data}

Python provides a \index{dictionary}\textbf{dictionary} data type that can be used for
mapping between arbitrary types.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

A Python dictionary is somewhat like a linguistic dictionary
— they both give you a systematic means of looking things up,
and so there is some potential for confusion. However, we hope that
it will usually be clear from the context which kind of dictionary
we are talking about.
\end{boxedminipage}
\end{sffamily}
\end{center}

Here we define \texttt{pos} to be an empty dictionary and then add three
entries to it, specifying the part-of-speech of some words.  We add
entries to a dictionary using the familiar square bracket notation:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }pos = {\{}{\}}
\pysrcprompt{>{}>{}> }pos{[}\pysrcstring{'colorless'}{]} = \pysrcstring{'adj'}
\pysrcprompt{>{}>{}> }pos{[}\pysrcstring{'furiously'}{]} = \pysrcstring{'adv'}
\pysrcprompt{>{}>{}> }pos{[}\pysrcstring{'ideas'}{]} = \pysrcstring{'n'}
\pysrcprompt{>{}>{}>}}\end{alltt}

So, for example, \texttt{pos{\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}colorless{\linebreak[0]}'}{]}{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} \pysrcstring{'{\linebreak[0]}adj{\linebreak[0]}'}} says that the look-up
value of \texttt{\pysrcstring{'{\linebreak[0]}colorless{\linebreak[0]}'}} in \texttt{pos} is the string \texttt{\pysrcstring{'{\linebreak[0]}adj{\linebreak[0]}'}}.
% Monkey-patching to get our dict examples to print consistently:
% 
% >>> from nltk import SortedDict
% >>> pos = SortedDict(pos) 

To look up a value in \texttt{pos}, we again use indexing notation, except
now the thing inside the square brackets is the item whose value we
want to recover:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }pos{[}\pysrcstring{'ideas'}{]}
\pysrcoutput{'n'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }pos{[}\pysrcstring{'colorless'}{]}
\pysrcoutput{'adj'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

The item used for look-up is called the \index{key}\textbf{key}, and the
data that is returned is known as the \index{value}\textbf{value}.  As with indexing
a list or string, we get an exception when we try to access the value
of a key that does not exist:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }pos{[}\pysrcstring{'missing'}{]}
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{  File "<stdin>", line 1, in ?}
\pysrcexcept{KeyError: 'missing'}
\pysrcexcept{}\pysrcprompt{>{}>{}>}}\end{alltt}

This raises an important question.  Unlike lists and strings, where we
can use \texttt{len{\linebreak[0]}({\linebreak[0]})} to work out which integers will be legal indices, how
do we work out the legal keys for a dictionary?  Fortunately, we can
check whether a key exists in a dictionary using the \texttt{\pysrckeyword{in}} operator:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrcstring{'colorless'} \pysrckeyword{in} pos
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'missing'} \pysrckeyword{in} pos
\pysrcoutput{False}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'missing'} \pysrckeyword{not} \pysrckeyword{in} pos
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Notice that we can use \texttt{\pysrckeyword{not} \pysrckeyword{in}} to check if a key is \docutilsroleemphasis{missing}.  Be
careful with the \texttt{\pysrckeyword{in}} operator for dictionaries: it only applies to
the keys and not their values.  If we check for a value, e.g. \texttt{\pysrcstring{'{\linebreak[0]}adj{\linebreak[0]}'}
\pysrckeyword{in} {\linebreak[0]}pos}, the result is \texttt{False}, since \texttt{\pysrcstring{'{\linebreak[0]}adj{\linebreak[0]}'}} is not a key.
We can loop over all the entries in a dictionary using a \texttt{\pysrckeyword{for}} loop.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} pos:
\pysrcmore{... }    \pysrckeyword{print} \pysrcstring{"{\%}s ({\%}s)"} {\%} (word, pos{[}word{]})
\pysrcmore{...}
\pysrcoutput{colorless (adj)}
\pysrcoutput{furiously (adv)}
\pysrcoutput{ideas (n)}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

We can see what the contents of the dictionary look like by inspecting
the variable \texttt{pos}.  Note the presence of the colon character to separate
each key from its corresponding value:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }pos
\pysrcoutput{{\{}'furiously': 'adv', 'ideas': 'n', 'colorless': 'adj'{\}}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Here, the contents of the dictionary are shown as \index{key-value
pairs}\textbf{key-value
pairs}.  As you can see,  the order of the key-value pairs is different
from the order in which they were originally entered.  This is because
dictionaries are not sequences but mappings. The keys in a mapping
are not inherently ordered, and any ordering that we might want to impose on the keys
exists independently of the mapping.  As we shall see later, this
gives us a lot of flexibility.

We can use the same key-value pair format to create a dictionary:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }pos = {\{}\pysrcstring{'furiously'}: \pysrcstring{'adv'}, \pysrcstring{'ideas'}: \pysrcstring{'n'}, \pysrcstring{'colorless'}: \pysrcstring{'adj'}{\}}
\pysrcprompt{>{}>{}>}}\end{alltt}
% Monkey-patching to get our dict examples to print consistently:
% 
% >>> pos = SortedDict(pos) 

Using the dictionary methods \texttt{\pysrcbuiltin{keys}({\linebreak[0]})}, \texttt{\pysrcbuiltin{values}({\linebreak[0]})} and \texttt{\pysrcbuiltin{items}({\linebreak[0]})},
we can access the keys and values as separate lists,
and also the key-value pairs:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }pos.keys()
\pysrcoutput{{[}'colorless', 'furiously', 'ideas'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }pos.values()
\pysrcoutput{{[}'adj', 'adv', 'n'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }pos.items()
\pysrcoutput{{[}('colorless', 'adj'), ('furiously', 'adv'), ('ideas', 'n'){]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{for} (key, val) \pysrckeyword{in} pos.items():
\pysrcmore{... }    \pysrckeyword{print} \pysrcstring{"{\%}s ==> {\%}s"} {\%} (key, val)
\pysrcmore{...}
\pysrcoutput{colorless ==> adj}
\pysrcoutput{furiously ==> adv}
\pysrcoutput{ideas ==> n}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Note that keys are forced to be unique.
Suppose we try to use a dictionary to store the fact that the
word \docutilsroleexample{content} is both a noun and a verb:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }pos{[}\pysrcstring{'content'}{]} = \pysrcstring{'n'}
\pysrcprompt{>{}>{}> }pos{[}\pysrcstring{'content'}{]} = \pysrcstring{'v'}
\pysrcprompt{>{}>{}> }pos
\pysrcoutput{{\{}'content': 'v', 'furiously': 'adv', 'ideas': 'n', 'colorless': 'adj'{\}}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Initially, \texttt{pos{\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}content{\linebreak[0]}'}{]}} is given the value \texttt{\pysrcstring{'{\linebreak[0]}n{\linebreak[0]}'}}, and this is
immediately overwritten with the new value \texttt{\pysrcstring{'{\linebreak[0]}v{\linebreak[0]}'}}.
In other words, there is only one entry for \texttt{\pysrcstring{'{\linebreak[0]}content{\linebreak[0]}'}}.
If we wanted to store multiple values in that entry, we could use a list,
e.g. \texttt{pos{\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}content{\linebreak[0]}'}{]}{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}n{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}v{\linebreak[0]}'}{]}}.


%___________________________________________________________________________

\subsection{Counting with Dictionaries}
\label{counting-with-dictionaries}

The values stored in a dictionary can be any kind of object, not just
a string — the values can even be dictionaries.  The most common
kind is actually an integer.  It turns out that we can use a
dictionary to store \index{counters}\textbf{counters} for many kinds of data.  For
instance, we can have a counter for all the letters of the alphabet;
each time we get a certain letter we increment its corresponding
counter:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }phrase = \pysrcstring{'colorless green ideas sleep furiously'}
\pysrcprompt{>{}>{}> }count = {\{}{\}}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} letter \pysrckeyword{in} phrase:
\pysrcmore{... }    \pysrckeyword{if} letter \pysrckeyword{not} \pysrckeyword{in} count:
\pysrcmore{... }        count{[}letter{]} = 0
\pysrcmore{... }    count{[}letter{]} += 1
\pysrcprompt{>{}>{}> }count
\pysrcoutput{{\{}'a': 1, ' ': 4, 'c': 1, 'e': 6, 'd': 1, 'g': 1, 'f': 1, 'i': 2,}
\pysrcoutput{ 'l': 4, 'o': 3, 'n': 1, 'p': 1, 's': 5, 'r': 3, 'u': 2, 'y': 1{\}}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Observe that \texttt{\pysrckeyword{in}} is used here in two different ways:
\texttt{\pysrckeyword{for} {\linebreak[0]}letter{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}phrase} iterates over every letter, running
the body of the \texttt{\pysrckeyword{for}} loop.  Inside this loop, the
conditional expression \texttt{\pysrckeyword{if} {\linebreak[0]}letter{\linebreak[0]} \pysrckeyword{not} \pysrckeyword{in} {\linebreak[0]}count} checks whether the letter is missing
from the dictionary.  If it is missing, we create a new entry and
set its value to zero: \texttt{count{\linebreak[0]}{[}{\linebreak[0]}letter{\linebreak[0]}{]}{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}0}.  Now we are sure
that the entry exists, and it may have a zero or non-zero value.
We finish the body of the \texttt{\pysrckeyword{for}} loop by incrementing this particular
counter using the \texttt{+{\linebreak[0]}=} assignment operator.
Finally, we print the dictionary, to see the letters and their counts.
This method of maintaining many counters will find many uses, and you
will become very familiar with it.  To make counting much easier, we
can use \texttt{defaultdict}, a special kind of container introduced in Python 2.5.
This is also included in NLTK for the benefit of readers who are using
Python 2.4, and can be imported as shown below.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }phrase = \pysrcstring{'colorless green ideas sleep furiously'}
\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk \pysrckeyword{import} defaultdict
\pysrcprompt{>{}>{}> }count = defaultdict(int)
\pysrcprompt{>{}>{}> }\pysrckeyword{for} letter \pysrckeyword{in} phrase:
\pysrcmore{... }    count{[}letter{]} += 1
\pysrcprompt{>{}>{}> }count
\pysrcoutput{{\{}'a': 1, ' ': 4, 'c': 1, 'e': 6, 'd': 1, 'g': 1, 'f': 1, 'i': 2,}
\pysrcoutput{ 'l': 4, 'o': 3, 'n': 1, 'p': 1, 's': 5, 'r': 3, 'u': 2, 'y': 1{\}}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Calling \texttt{defaultdict{\linebreak[0]}({\linebreak[0]}int{\linebreak[0]})} creates a special kind of dictionary.
When that dictionary is accessed with a non-existent key
— i.e. the first time a particular letter is encountered —
then \texttt{int{\linebreak[0]}({\linebreak[0]})} is called to produce the initial value for this key (i.e. \texttt{0}).
You can test this by running the above code, then typing \texttt{count{\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}X{\linebreak[0]}'}{]}}
and seeing that it returns a zero value (and not a \texttt{KeyError} as in the
case of normal Python dictionaries).
The function defaultdict is very handy and will be used in many places later on.
\end{boxedminipage}
\end{sffamily}
\end{center}

There are other useful ways to display the result, such as sorting
alphabetically by the letter:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sorted(count.items())
\pysrcoutput{{[}(' ', 4), ('a', 1), ('c', 1), ('d', 1), ('e', 6), ('f', 1), ...,}
\pysrcoutput{...('y', 1){]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

The function \texttt{sorted{\linebreak[0]}({\linebreak[0]})} is similar to the \texttt{sort{\linebreak[0]}({\linebreak[0]})}
method on sequences, but rather than sorting in-place, it produces
a new sorted copy of its argument. Moreover, as we will see very
soon, \texttt{sorted{\linebreak[0]}({\linebreak[0]})} will work on a wider variety of data types,
including dictionaries.
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\subsection{Getting Unique Entries}
\label{getting-unique-entries}

Sometimes, we don't want to count at all, but just want to make a
record of the items that we have seen, regardless of repeats.  For
example, we might want to compile a vocabulary from a document.  This
is a sorted list of the words that appeared, regardless of frequency.
At this stage we have two ways to do this.  The first uses lists,
while the second uses sets.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sentence = \pysrcstring{"she sells sea shells by the sea shore"}.split()
\pysrcprompt{>{}>{}> }words = {[}{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} sentence:
\pysrcmore{... }    \pysrckeyword{if} word \pysrckeyword{not} \pysrckeyword{in} words:
\pysrcmore{... }        words.append(word)
\pysrcmore{...}
\pysrcprompt{>{}>{}> }sorted(words)
\pysrcoutput{{[}'by', 'sea', 'sells', 'she', 'shells', 'shore', 'the'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

There is a better way to do this task using Python's
\index{set}\textbf{set} data type.  We can convert \texttt{sentence} into a set, using
\texttt{set{\linebreak[0]}({\linebreak[0]}sentence{\linebreak[0]})}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }set(sentence)
\pysrcoutput{set({[}'shells', 'sells', 'shore', 'she', 'sea', 'the', 'by'{]})}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

The order of items in a set is not significant, and they will usually
appear in a different order to the one they were entered in.  The
main point here is that converting a list to a set removes any duplicates.
We convert it back into a list, sort it, and print.  Here is the complete
program:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sentence = \pysrcstring{"she sells sea shells by the sea shore"}.split()
\pysrcprompt{>{}>{}> }sorted(set(sentence))
\pysrcoutput{{[}'by', 'sea', 'sells', 'she', 'shells', 'shore', 'the'{]}}}\end{alltt}

Here we have seen that there is sometimes more than one way to solve a
problem with a program.  In this case, we used three different
built-in data types, a list, a dictionary, and a set.  The \texttt{set} data type
mostly closely modeled our task, so it required the least amount of work.


%___________________________________________________________________________

\subsection{Scaling Up}
\label{scaling-up}

We can use dictionaries to count word occurrences.  For example, the
following code uses NLTK's corpus reader to load
\emph{Macbeth} and count the frequency of each word.
Before we can use NLTK we need to tell Python to load it, using
the statement \texttt{\pysrckeyword{import} {\linebreak[0]}nltk}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} nltk
\pysrcprompt{>{}>{}> }count = nltk.defaultdict(int)                     \pysrccomment{{\#} initialize a dictionary}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} nltk.corpus.gutenberg.words(\pysrcstring{'shakespeare-macbeth.txt'}): \pysrccomment{{\#} tokenize Macbeth}
\pysrcmore{... }    word = word.lower()                           \pysrccomment{{\#} normalize to lowercase}
\pysrcmore{... }    count{[}word{]} += 1                              \pysrccomment{{\#} increment the counter}
\pysrcmore{...}
\pysrcprompt{>{}>{}>}}\end{alltt}

You will learn more about accessing corpora in \href{\#sec-extracting-text-from-corpora}{Section 2.2.3}.
For now, you just need to know that \texttt{gutenberg{\linebreak[0]}.{\linebreak[0]}words{\linebreak[0]}({\linebreak[0]})} returns a list of words,
in this case from Shakespeare's play \emph{Macbeth}, and we are iterating over this list
using a \texttt{\pysrckeyword{for}} loop.  We convert each word to lowercase using the string
method \texttt{word{\linebreak[0]}.{\linebreak[0]}lower{\linebreak[0]}({\linebreak[0]})}, and use a dictionary to maintain a set of
counters, one per word.  Now we can inspect the contents of the
dictionary to get counts for particular words:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }count{[}\pysrcstring{'scotland'}{]}
\pysrcoutput{12}
\pysrcoutput{}\pysrcprompt{>{}>{}> }count{[}\pysrcstring{'the'}{]}
\pysrcoutput{692}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}


%___________________________________________________________________________

\subsection{Exercises}
\label{id5}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☺ Review the mappings in \href{\#linguistic-objects}{Table 1.3}.  Discuss any other
examples of mappings you can think of.  What type of information do they map
from and to?

\item {} 
☼ Using the Python interpreter in interactive mode, experiment with
the examples in this section.  Create a dictionary \texttt{d}, and add
some entries.  What happens if you try to access a non-existent
entry, e.g. \texttt{d{\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}xyz{\linebreak[0]}'}{]}}?

\item {} 
☼ Try deleting an element from a dictionary, using the syntax
\texttt{\pysrckeyword{del} {\linebreak[0]}d{\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}abc{\linebreak[0]}'}{]}}.  Check that the item was deleted.

\item {} 
☼ Create a dictionary \texttt{e}, to represent a single lexical entry
for some word of your choice.
Define keys like \texttt{headword}, \texttt{part{\linebreak[0]}-{\linebreak[0]}of{\linebreak[0]}-{\linebreak[0]}speech}, \texttt{sense}, and
\texttt{example}, and assign them suitable values.

\item {} 
☼ Create two dictionaries, \texttt{d1} and \texttt{d2}, and add some entries to
each.  Now issue the command \texttt{d1{\linebreak[0]}.{\linebreak[0]}update{\linebreak[0]}({\linebreak[0]}d2{\linebreak[0]})}.  What did this do?
What might it be useful for?

\item {} 
◑ Write a program that takes a sentence expressed as a single string,
splits it and counts up the words.  Get it to print out each word and the
word's frequency, one per line, in alphabetical order.

\end{list}


%___________________________________________________________________________

\section{Regular Expressions}
\label{regular-expressions}

For a moment, imagine that you are editing a large text, and you have
strong dislike of repeated occurrences of the word \docutilsroleexample{very}.
How could you find all such cases in the text? To be concrete, let's
suppose that we assign the following text to the variable \texttt{s}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }s = \pysrcstring{"{}"{}"Google Analytics is very very very nice (now)}
\pysrcmore{... }\pysrcstring{By Jason Hoffman 18 August 06}
\pysrcmore{... }\pysrcstring{Google Analytics, the result of Google's acquisition of the San}
\pysrcmore{... }\pysrcstring{Diego-based Urchin Software Corporation, really really opened its}
\pysrcmore{... }\pysrcstring{doors to the world a couple of days ago, and it allows you to}
\pysrcmore{... }\pysrcstring{track up to 10 sites within a single google account.}
\pysrcmore{... }\pysrcstring{"{}"{}"}
\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent} Python's triple quotes \texttt{\pysrcstring{"{\linebreak[0]}"}"} are used here since they allow us
to break a string across lines.

One approach to our task would be to convert the string into a list,
and look for adjacent items that are both equal to the string
\texttt{\pysrcstring{'{\linebreak[0]}very{\linebreak[0]}'}}. We use the \texttt{range{\linebreak[0]}({\linebreak[0]}n{\linebreak[0]})} function in this example to create
a list of consecutive integers from 0 up to, but not including, \texttt{n}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text = s.split()
\pysrcprompt{>{}>{}> }\pysrckeyword{for} n \pysrckeyword{in} range(len(text)):
\pysrcmore{... }   \pysrckeyword{if} text{[}n{]} == \pysrcstring{'very'} \pysrckeyword{and} text{[}n+1{]} == \pysrcstring{'very'}:
\pysrcmore{... }                \pysrckeyword{print} n, n+1
\pysrcmore{...}
\pysrcoutput{3 4}
\pysrcoutput{4 5}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent} However, such an approach is not very flexible or convenient. In this
section, we will present Python's \index{regular expression}\textbf{regular expression} module
\texttt{re}, which supports powerful search and substitution inside
strings.  As a gentle introduction, we will start out using a utility
function \texttt{re{\_}show{\linebreak[0]}({\linebreak[0]})} to illustrate how regular expressions match
against substrings. \texttt{re{\_}show{\linebreak[0]}({\linebreak[0]})} takes two arguments, a pattern that
it is looking for, and a string in which the pattern might occur.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} nltk
\pysrcprompt{>{}>{}> }nltk.re{\_}show(\pysrcstring{'very very'}, s)
\pysrcoutput{Google Analytics is {\{}very very{\}} very nice (now)}
\pysrcoutput{...}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent} (We have only displayed the first part of \texttt{s} that
is returned, since the rest is irrelevant for the moment.)
As you can see, \texttt{re{\_}show} places curly braces around the first
occurrence it has found of the string \texttt{\pysrcstring{'{\linebreak[0]}very{\linebreak[0]} {\linebreak[0]}very{\linebreak[0]}'}}. So an important
part of what \texttt{re{\_}show} is doing is searching for any substring of
\texttt{s} that \index{matches}\textbf{matches} the pattern in its first argument.

Now we might want to modify the example so that \texttt{re{\_}show} highlights
cases where there are two \docutilsroleemphasis{or more} adjacent sequences of
\texttt{\pysrcstring{'{\linebreak[0]}very{\linebreak[0]}'}}. To do this, we need to use a \index{regular expression
operator}\textbf{regular expression
operator}, namely \texttt{\pysrcstring{'{\linebreak[0]}+{\linebreak[0]}'}}. If \docutilsroleplaceholder{s} is a string, then \docutilsroleplaceholder{s}\texttt{+} means: 'match one or more occurrences of \docutilsroleplaceholder{s}'. Let's first look
at the case where \docutilsroleplaceholder{s} is a single character, namely the letter
\texttt{\pysrcstring{'{\linebreak[0]}o{\linebreak[0]}'}}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.re{\_}show(\pysrcstring{'o+'}, s)
\pysrcoutput{G{\{}oo{\}}gle Analytics is very very very nice (n{\{}o{\}}w)}
\pysrcoutput{...}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent} \texttt{\pysrcstring{'{\linebreak[0]}o{\linebreak[0]}+{\linebreak[0]}'}} is our first proper regular expression. You can think of it as
matching an \docutilsroleemphasis{infinite set} of strings, namely the set {\{}\texttt{\pysrcstring{'{\linebreak[0]}o{\linebreak[0]}'}},
\texttt{\pysrcstring{'{\linebreak[0]}oo{\linebreak[0]}'}}, \texttt{\pysrcstring{'{\linebreak[0]}ooo{\linebreak[0]}'}}, ...{\}}. But we would really like to match
sequences of least two \texttt{\pysrcstring{'{\linebreak[0]}o{\linebreak[0]}'}}s; for this, we need
the regular expression  \texttt{\pysrcstring{'{\linebreak[0]}oo{\linebreak[0]}+{\linebreak[0]}'}}, which matches any string consisting of
\texttt{\pysrcstring{'{\linebreak[0]}o{\linebreak[0]}'}} followed by one or more occurrences of \texttt{o}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.re{\_}show(\pysrcstring{'oo+'}, s)
\pysrcoutput{G{\{}oo{\}}gle Analytics is very very very nice (now)}
\pysrcoutput{...}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Let's return to the task of identifying multiple occurrences of
\texttt{\pysrcstring{'{\linebreak[0]}very{\linebreak[0]}'}}. Some initially plausible candidates won't do what we
want. For example, \texttt{\pysrcstring{'{\linebreak[0]}very{\linebreak[0]}+{\linebreak[0]}'}} would match \texttt{\pysrcstring{'{\linebreak[0]}veryyy{\linebreak[0]}'}} (but not
\texttt{\pysrcstring{'{\linebreak[0]}very{\linebreak[0]} {\linebreak[0]}very{\linebreak[0]}'}}), since the \texttt{+} scopes over the immediately
preceding expression, in this case \texttt{\pysrcstring{'{\linebreak[0]}y{\linebreak[0]}'}}. To widen the scope of
\texttt{+}, we need to use parentheses, as in \texttt{\pysrcstring{'{\linebreak[0]}({\linebreak[0]}very{\linebreak[0]}){\linebreak[0]}+{\linebreak[0]}'}}. Will this
match \texttt{\pysrcstring{'{\linebreak[0]}very{\linebreak[0]} {\linebreak[0]}very{\linebreak[0]}'}}? No, because we've forgotten about the
whitespace between the two words; instead, it will match strings like
\texttt{\pysrcstring{'{\linebreak[0]}veryvery{\linebreak[0]}'}}. However, the following \docutilsroleemphasis{does} work:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.re{\_}show(\pysrcstring{'(very{\textbackslash}s)+'}, s)
\pysrcoutput{Google Analytics is {\{}very very very {\}}nice (now)}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent} Characters preceded by a \texttt{{\textbackslash}}, such as \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}s{\linebreak[0]}'}}, have a
special interpretation inside regular expressions; thus, \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}s{\linebreak[0]}'}}
matches a whitespace character. We could have used \texttt{\pysrcstring{'{\linebreak[0]} {\linebreak[0]}'}} in our
pattern, but \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}s{\linebreak[0]}'}} is better practice in general. One reason is
that the sense of ``whitespace'' we are using is more general than you
might have imagined; it includes not just inter-word spaces, but also
tabs and newlines. If you try to inspect the variable \texttt{s}, you
might initially get a shock:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }s
\pysrcoutput{"Google Analytics is very very very nice (now){\textbackslash}nBy Jason Hoffman}
\pysrcoutput{18 August 06{\textbackslash}nGoogle}
\pysrcoutput{...}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent} You might recall that \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}n{\linebreak[0]}'}} is a special character that corresponds
to a newline in a string. The following example shows how newline is
matched by \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}s{\linebreak[0]}'}}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }s2 = \pysrcstring{"I'm very very{\textbackslash}nvery happy"}
\pysrcprompt{>{}>{}> }nltk.re{\_}show(\pysrcstring{'very{\textbackslash}s'}, s2)
\pysrcoutput{I'm {\{}very {\}}{\{}very}
\pysrcoutput{{\}}{\{}very {\}}happy}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Python's \texttt{re{\linebreak[0]}.{\linebreak[0]}findall{\linebreak[0]}(}\docutilsroleplaceholder{patt}, \docutilsroleplaceholder{s}\texttt{)} function is a useful
way to find all the substrings in \docutilsroleplaceholder{s} that are matched by \docutilsroleplaceholder{patt}.
Before illustrating, let's introduce  two further
special characters, \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}d{\linebreak[0]}'}} and \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}w{\linebreak[0]}'}}: the first
will match any digit, and the second will match any alphanumeric character.
Before we can use \texttt{re{\linebreak[0]}.{\linebreak[0]}findall{\linebreak[0]}({\linebreak[0]})} we have to load Python's regular
expression module, using \texttt{\pysrckeyword{import} {\linebreak[0]}re}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} re
\pysrcprompt{>{}>{}> }re.findall(\pysrcstring{'{\textbackslash}d{\textbackslash}d'}, s)
\pysrcoutput{{[}'18', '06', '10'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }re.findall(\pysrcstring{'{\textbackslash}s{\textbackslash}w{\textbackslash}w{\textbackslash}w{\textbackslash}s'}, s)
\pysrcoutput{{[}' the ', ' the ', ' its{\textbackslash}n', ' the ', ' and ', ' you '{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent} As you will see, the second example matches three-letter
words. However, this regular expression is not quite what we
want. First, the leading and trailing spaces are extraneous. Second,
it will fail to match against strings such as \texttt{\pysrcstring{'{\linebreak[0]}the{\linebreak[0]} {\linebreak[0]}San{\linebreak[0]}'}}, where two
three-letter words are adjacent. To solve this problem, we can use
another special character, namely \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}b{\linebreak[0]}'}}. This is sometimes called a
``zero-width'' character; it matches against the empty string, but only
at the beginning and end of words:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }re.findall(r\pysrcstring{'{\textbackslash}b{\textbackslash}w{\textbackslash}w{\textbackslash}w{\textbackslash}b'}, s)
\pysrcoutput{{[}'now', 'the', 'the', 'San', 'its', 'the', 'ago', 'and', 'you'{]}}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

This example uses a Python \index{raw string}\textbf{raw string}:
\texttt{r\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}b{\linebreak[0]}{\textbackslash}{\linebreak[0]}w{\linebreak[0]}{\textbackslash}{\linebreak[0]}w{\linebreak[0]}{\textbackslash}{\linebreak[0]}w{\linebreak[0]}{\textbackslash}{\linebreak[0]}b{\linebreak[0]}'}}. The specific justification here is that in an
ordinary string, \texttt{{\textbackslash}{\linebreak[0]}b} is interpreted as a backspace character.
Python will convert it to a backspace in a regular expression
unless you use the \texttt{r} prefix to create a raw string as shown
above. Another use for raw strings is to match strings that
include backslashes. Suppose we want to match 'either{\textbackslash}or'. In order
to create a regular expression, the backslash needs to be escaped,
since it is a special character;
so we want to pass the pattern \texttt{{\textbackslash}{\linebreak[0]}{\textbackslash}} to the regular expression
interpreter. But to express this as a Python string literal, each
backslash must be escaped again, yielding the string
\texttt{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}{\textbackslash}{\linebreak[0]}{\textbackslash}{\linebreak[0]}{\textbackslash}{\linebreak[0]}'}. However, with a raw string, this reduces down to
\texttt{r{\linebreak[0]}'{\linebreak[0]}{\textbackslash}{\linebreak[0]}{\textbackslash}{\linebreak[0]}'}.
\end{boxedminipage}
\end{sffamily}
\end{center}

Returning to the case of repeated words, we might want to look for
cases involving \texttt{\pysrcstring{'{\linebreak[0]}very{\linebreak[0]}'}} or \texttt{\pysrcstring{'{\linebreak[0]}really{\linebreak[0]}'}}, and for this we use the
disjunction operator \texttt{|}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.re{\_}show(\pysrcstring{'((very|really){\textbackslash}s)+'}, s)
\pysrcoutput{Google Analytics is {\{}very very very {\}}nice (now)}
\pysrcoutput{By Jason Hoffman 18 August 06}
\pysrcoutput{Google Analytics, the result of Google's acquisition of the San}
\pysrcoutput{Diego-based Urchin Software Corporation, {\{}really really {\}}opened its}
\pysrcoutput{doors to the world a couple of days ago, and it allows you to}
\pysrcoutput{track up to 10 sites within a single google account.}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

In addition to the matches just illustrated, the regular expression
\texttt{\pysrcstring{'{\linebreak[0]}({\linebreak[0]}({\linebreak[0]}very{\linebreak[0]}|{\linebreak[0]}really{\linebreak[0]}){\linebreak[0]}{\textbackslash}{\linebreak[0]}s{\linebreak[0]}){\linebreak[0]}+{\linebreak[0]}'}} will also match cases where the two disjuncts
occur with each other, such as the string \texttt{\pysrcstring{'{\linebreak[0]}really{\linebreak[0]} {\linebreak[0]}very{\linebreak[0]} {\linebreak[0]}really{\linebreak[0]} {\linebreak[0]}'}}.

Let's now look at how to perform substitutions, using the \texttt{re{\linebreak[0]}.{\linebreak[0]}sub{\linebreak[0]}({\linebreak[0]})}
function.  In the first instance we replace
all instances of \texttt{l} with \texttt{s}.  Note that this generates a string
as output, and doesn't modify the original string.  Then we replace
any instances of \texttt{green} with \texttt{red}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = \pysrcstring{"colorless green ideas sleep furiously"}
\pysrcprompt{>{}>{}> }re.sub(\pysrcstring{'l'}, \pysrcstring{'s'}, sent)
\pysrcoutput{'cosorsess green ideas sseep furioussy'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }re.sub(\pysrcstring{'green'}, \pysrcstring{'red'}, sent)
\pysrcoutput{'colorless red ideas sleep furiously'}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

We can also disjoin individual characters using a square bracket
notation.  For example, \texttt{{[}{\linebreak[0]}aeiou{\linebreak[0]}{]}} matches any of \texttt{a}, \texttt{e},
\texttt{i}, \texttt{o}, or \texttt{u}, that is, any vowel.  The expression
\texttt{{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]}aeiou{\linebreak[0]}{]}} matches any single character that is \docutilsroleemphasis{not} a vowel.
In the following example, we match sequences consisting of a non-vowel
followed by a vowel.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.re{\_}show(\pysrcstring{'{[}{\textasciicircum}aeiou{]}{[}aeiou{]}'}, sent)
\pysrcoutput{{\{}co{\}}{\{}lo{\}}r{\{}le{\}}ss g{\{}re{\}}en{\{} i{\}}{\{}de{\}}as s{\{}le{\}}ep {\{}fu{\}}{\{}ri{\}}ously}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent}
Using the same regular expression, the function \texttt{re{\linebreak[0]}.{\linebreak[0]}findall{\linebreak[0]}({\linebreak[0]})}
returns a list of all the substrings in \texttt{sent} that are matched:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }re.findall(\pysrcstring{'{[}{\textasciicircum}aeiou{]}{[}aeiou{]}'}, sent)
\pysrcoutput{{[}'co', 'lo', 'le', 're', ' i', 'de', 'le', 'fu', 'ri'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}


%___________________________________________________________________________

\subsection{Groupings}
\label{groupings}

Returning briefly to our earlier problem with unwanted whitespace
around three-letter words, we note that \texttt{re{\linebreak[0]}.{\linebreak[0]}findall{\linebreak[0]}({\linebreak[0]})} behaves
slightly differently if we create \index{groups}\textbf{groups} in the regular
expression using parentheses; it only returns strings that occur
within the groups:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }re.findall(\pysrcstring{'{\textbackslash}s({\textbackslash}w{\textbackslash}w{\textbackslash}w){\textbackslash}s'}, s)
\pysrcoutput{{[}'the', 'the', 'its', 'the', 'and', 'you'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent}
The same device allows us to select only the non-vowel characters
that appear before a vowel:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }re.findall(\pysrcstring{'({[}{\textasciicircum}aeiou{]}){[}aeiou{]}'}, sent)
\pysrcoutput{{[}'c', 'l', 'l', 'r', ' ', 'd', 'l', 'f', 'r'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

By delimiting a second group in the regular expression, we can even
generate pairs (or \index{tuples}\textbf{tuples}) that we may then go on and
tabulate.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }re.findall(\pysrcstring{'({[}{\textasciicircum}aeiou{]})({[}aeiou{]})'}, sent)
\pysrcoutput{{[}('c', 'o'), ('l', 'o'), ('l', 'e'), ('r', 'e'), (' ', 'i'),}
\pysrcoutput{    ('d', 'e'), ('l', 'e'), ('f', 'u'), ('r', 'i'){]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Our next example also makes use of groups. One further special
character is the so-called wildcard element, \texttt{\pysrcstring{'{\linebreak[0]}.{\linebreak[0]}'}}; this has the
distinction of matching any single character (except \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}n{\linebreak[0]}'}}). Given
the string \texttt{s3}, our task is to pick out login names and email domains:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }s3 = \pysrcstring{"{}"{}"}
\pysrcmore{... }\pysrcstring{<hart@vmd.cso.uiuc.edu>}
\pysrcmore{... }\pysrcstring{Final editing was done by Martin Ward <Martin.Ward@uk.ac.durham>}
\pysrcmore{... }\pysrcstring{Michael S. Hart <hart@pobox.com>}
\pysrcmore{... }\pysrcstring{Prepared by David Price, email <ccx074@coventry.ac.uk>"{}"{}"}}\end{alltt}

The task is made much easier by the fact that all the email addresses
in the example are delimited by angle brackets, and we can exploit
this feature in our regular expression:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }re.findall(r\pysrcstring{'<(.+)@(.+)>'}, s3)
\pysrcoutput{{[}('hart', 'vmd.cso.uiuc.edu'), ('Martin.Ward', 'uk.ac.durham'),}
\pysrcoutput{('hart', 'pobox.com'), ('ccx074', 'coventry.ac.uk'){]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent}
Since \texttt{\pysrcstring{'{\linebreak[0]}.{\linebreak[0]}'}} matches any single character, \texttt{\pysrcstring{'{\linebreak[0]}.{\linebreak[0]}+{\linebreak[0]}'}} will match any
non-empty \docutilsroleemphasis{string} of characters, including punctuation symbols
such as the period.

One question that might occur to you is how do
we specify a match against a period? The answer is that we have to
place a \texttt{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}'} immediately before the \texttt{\pysrcstring{'{\linebreak[0]}.{\linebreak[0]}'}} in order to escape its
special interpretation.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }re.findall(r\pysrcstring{'({\textbackslash}w+{\textbackslash}.)'}, s3)
\pysrcoutput{{[}'vmd.', 'cso.', 'uiuc.', 'Martin.', 'uk.', 'ac.', 'S.',}
\pysrcoutput{'pobox.', 'coventry.', 'ac.'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

Now, let's suppose that we wanted to match occurrences of both
\texttt{\pysrcstring{'{\linebreak[0]}Google{\linebreak[0]}'}} and \texttt{\pysrcstring{'{\linebreak[0]}google{\linebreak[0]}'}} in our sample text. If you have been
following up till now, you would reasonably expect that this regular
expression with a disjunction would do the trick:
\texttt{\pysrcstring{'{\linebreak[0]}({\linebreak[0]}G{\linebreak[0]}|{\linebreak[0]}g{\linebreak[0]}){\linebreak[0]}oogle{\linebreak[0]}'}}. But look what happens when we try this with
\texttt{re{\linebreak[0]}.{\linebreak[0]}findall{\linebreak[0]}({\linebreak[0]})}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }re.findall(\pysrcstring{'(G|g)oogle'}, s)
\pysrcoutput{{[}'G', 'G', 'G', 'g'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent}
What is going wrong? We innocently used
the parentheses to indicate the scope of the operator \texttt{\pysrcstring{'{\linebreak[0]}|{\linebreak[0]}'}}, but
\texttt{re{\linebreak[0]}.{\linebreak[0]}findall{\linebreak[0]}({\linebreak[0]})} has interpreted them as marking a group. In order to
tell \texttt{re{\linebreak[0]}.{\linebreak[0]}findall{\linebreak[0]}({\linebreak[0]})}  ``don't try to do anything special with these
parentheses'', we need an extra piece of notation:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }re.findall(\pysrcstring{'(?:G|g)oogle'}, s)
\pysrcoutput{{[}'Google', 'Google', 'Google', 'google'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent}
Placing \texttt{\pysrcstring{'{\linebreak[0]}?{\linebreak[0]}:{\linebreak[0]}'}} immediately after the opening parenthesis makes it
explicit that the parentheses are just being used for scoping.


%___________________________________________________________________________

\subsection{Practice Makes Perfect}
\label{practice-makes-perfect}

Regular expressions are very flexible and very powerful. However, they
often don't do what you expect. For this reason, you are strongly
encouraged to try out a variety of tasks using \texttt{re{\_}show{\linebreak[0]}({\linebreak[0]})} and
\texttt{re{\linebreak[0]}.{\linebreak[0]}findall{\linebreak[0]}({\linebreak[0]})} in order to develop your intuitions further; the
exercises below should help get you started.  We suggest that you build up a
regular expression in small pieces, rather than trying to get it
completely right first time.  Here are some operators
and sequences that are commonly used in natural language processing.
\begin{quote}

\leavevmode
\begin{longtable}[c]{|p{0.143\locallinewidth}|p{0.797\locallinewidth}|}
\hline
\multicolumn{2}{|l|}{
Commonly-used Operators and Sequences
} \\
\hline

\texttt{*}
 & 
Zero or more, e.g. \texttt{a{\linebreak[0]}*}, \texttt{{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}*}
 \\
\hline

\texttt{+}
 & 
One or more, e.g. \texttt{a{\linebreak[0]}+}, \texttt{{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}+}
 \\
\hline

\texttt{?}
 & 
Zero or one (i.e. optional), e.g. \texttt{a{\linebreak[0]}?}, \texttt{{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}?}
 \\
\hline

\texttt{{[}{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}{]}}
 & 
A set or range of characters, e.g. \texttt{{[}{\linebreak[0]}aeiou{\linebreak[0]}{]}}, \texttt{{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z0{\linebreak[0]}-{\linebreak[0]}9{\linebreak[0]}{]}}
 \\
\hline

\texttt{({\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]})}
 & 
Grouping parentheses, e.g. \texttt{({\linebreak[0]}the{\linebreak[0]}|{\linebreak[0]}a{\linebreak[0]}|{\linebreak[0]}an{\linebreak[0]})}
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}b}
 & 
Word boundary (zero width)
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}d}
 & 
Any decimal digit (\texttt{{\textbackslash}{\linebreak[0]}D} is any non-digit)
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}s}
 & 
Any whitespace character (\texttt{{\textbackslash}{\linebreak[0]}S} is any non-whitespace character)
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}w}
 & 
Any alphanumeric character (\texttt{{\textbackslash}{\linebreak[0]}W} is any non-alphanumeric character
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}t}
 & 
The tab character
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}n}
 & 
The newline character
 \\
\hline
\caption{}\end{longtable}
\end{quote}


%___________________________________________________________________________

\subsection{Exercises}
\label{id6}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Describe the class of strings matched by the following regular
expressions. Note that \texttt{\pysrcstring{'{\linebreak[0]}*{\linebreak[0]}'}} means: match zero or more
occurrences of the preceding regular expression.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
\texttt{{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}zA{\linebreak[0]}-{\linebreak[0]}Z{\linebreak[0]}{]}{\linebreak[0]}+}

\item {} 
\texttt{{[}{\linebreak[0]}A{\linebreak[0]}-{\linebreak[0]}Z{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}*}

\item {} 
\texttt{{\textbackslash}{\linebreak[0]}d{\linebreak[0]}+{\linebreak[0]}({\linebreak[0]}{\textbackslash}{\linebreak[0]}.{\linebreak[0]}{\textbackslash}{\linebreak[0]}d{\linebreak[0]}+{\linebreak[0]}){\linebreak[0]}?}

\item {} 
\texttt{({\linebreak[0]}{[}{\linebreak[0]}bcdfghjklmnpqrstvwxyz{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}aeiou{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}bcdfghjklmnpqrstvwxyz{\linebreak[0]}{]}{\linebreak[0]}){\linebreak[0]}*}

\item {} 
\texttt{{\textbackslash}{\linebreak[0]}w{\linebreak[0]}+{\linebreak[0]}|{\linebreak[0]}{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]}{\textbackslash}{\linebreak[0]}w{\linebreak[0]}{\textbackslash}{\linebreak[0]}s{\linebreak[0]}{]}{\linebreak[0]}+}

\end{list}

Test your answers using \texttt{re{\_}show{\linebreak[0]}({\linebreak[0]})}.

\item {} 
☼ Write regular expressions to match the following classes of strings:
\begin{quote}
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
A single determiner (assume that \docutilsroleexample{a}, \docutilsroleexample{an}, and \docutilsroleexample{the}
are the only determiners).

\item {} 
An arithmetic expression using integers, addition, and
multiplication, such as \texttt{2{\linebreak[0]}*{\linebreak[0]}3{\linebreak[0]}+{\linebreak[0]}8}.

\end{list}
\end{quote}

\item {} 
◑ The above example of extracting (name, domain) pairs from
text does not work when there is more than one email address
on a line, because the \texttt{+} operator is ``greedy'' and consumes
too much of the input.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Experiment with input text containing more than one email address
per line, such as that shown below.  What happens?

\item {} 
Using \texttt{re{\linebreak[0]}.{\linebreak[0]}findall{\linebreak[0]}({\linebreak[0]})}, write another regular expression
to extract email addresses, replacing the period character
with a range or negated range, such as \texttt{{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}+} or \texttt{{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]} {\linebreak[0]}>{\linebreak[0]}{]}{\linebreak[0]}+}.

\item {} 
Now try to match email addresses by changing the regular
expression \texttt{.{\linebreak[0]}+} to its ``non-greedy'' counterpart, \texttt{.{\linebreak[0]}+{\linebreak[0]}?}

\end{list}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }s = \pysrcstring{"{}"{}"}
\pysrcmore{... }\pysrcstring{austen-emma.txt:hart@vmd.cso.uiuc.edu  (internet)  hart@uiucvmd (bitnet)}
\pysrcmore{... }\pysrcstring{austen-emma.txt:Internet (72600.2026@compuserve.com); TEL: (212-254-5093)}
\pysrcmore{... }\pysrcstring{austen-persuasion.txt:Editing by Martin Ward (Martin.Ward@uk.ac.durham)}
\pysrcmore{... }\pysrcstring{blake-songs.txt:Prepared by David Price, email ccx074@coventry.ac.uk}
\pysrcmore{... }\pysrcstring{"{}"{}"}}\end{alltt}

\item {} 
◑ Write code to convert text into Pig Latin.  This involves two steps:
move any consonant (or consonant cluster) that appears at the start of the word
to the end, then append \docutilsroleexample{ay}, e.g. \docutilsroleexample{string} → \docutilsroleexample{ingstray},
\docutilsroleexample{idle} → \docutilsroleexample{idleay}.  \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}en{\linebreak[0]}.{\linebreak[0]}wikipedia{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}wiki{\linebreak[0]}/{\linebreak[0]}Pig{\_}Latin}

\item {} 
◑ Write code to convert text into \emph{hAck3r} again, this time using regular expressions
and substitution, where
\texttt{e} → \texttt{3},
\texttt{i} → \texttt{1},
\texttt{o} → \texttt{0},
\texttt{l} → \texttt{|},
\texttt{s} → \texttt{5},
\texttt{.} → \texttt{5w33t{\linebreak[0]}!},
\texttt{ate} → \texttt{8}.
Normalize the text to lowercase before converting it.
Add more substitutions of your own.  Now try to map
\texttt{s} to two different values: \texttt{{\$}} for word-initial \texttt{s},
and \texttt{5} for word-internal \texttt{s}.

\item {} 
★ Read the Wikipedia entry on \emph{Soundex}.  Implement this
algorithm in Python.

\end{list}


%___________________________________________________________________________

\section{Summary}
\label{summary}
\begin{itemize}
\item {} 
Text is represented in Python using strings, and we type these with
single or double quotes: \texttt{\pysrcstring{'{\linebreak[0]}Hello{\linebreak[0]}'}}, \texttt{\pysrcstring{"{\linebreak[0]}World{\linebreak[0]}"}}.

\item {} 
The characters of a string are accessed using indexes, counting from zero:
\texttt{\pysrcstring{'{\linebreak[0]}Hello{\linebreak[0]} {\linebreak[0]}World{\linebreak[0]}'}{[}{\linebreak[0]}1{\linebreak[0]}{]}} gives the value \texttt{e}.  The length of a string is
found using \texttt{len{\linebreak[0]}({\linebreak[0]})}.

\item {} 
Substrings are accessed using slice notation: \texttt{\pysrcstring{'{\linebreak[0]}Hello{\linebreak[0]} {\linebreak[0]}World{\linebreak[0]}'}{[}{\linebreak[0]}1{\linebreak[0]}:{\linebreak[0]}5{\linebreak[0]}{]}}
gives the value \texttt{ello}.  If the start index is omitted, the
substring begins at the start of the string; if the end index is omitted,
the slice continues to the end of the string.

\item {} 
Sequences of words are represented in Python using lists of strings:
\texttt{{[}\pysrcstring{'{\linebreak[0]}colorless{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}green{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}ideas{\linebreak[0]}'}{]}}.  We can use indexing, slicing
and the \texttt{len{\linebreak[0]}({\linebreak[0]})} function on lists.

\item {} 
Strings can be split into lists: \texttt{\pysrcstring{'{\linebreak[0]}Hello{\linebreak[0]} {\linebreak[0]}World{\linebreak[0]}'}.{\linebreak[0]}split{\linebreak[0]}({\linebreak[0]})} gives
\texttt{{[}\pysrcstring{'{\linebreak[0]}Hello{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}World{\linebreak[0]}'}{]}}.  Lists can be joined into strings:
\texttt{\pysrcstring{'{\linebreak[0]}/{\linebreak[0]}'}.{\linebreak[0]}join{\linebreak[0]}({\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}Hello{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}World{\linebreak[0]}'}{]}{\linebreak[0]})} gives \texttt{\pysrcstring{'{\linebreak[0]}Hello{\linebreak[0]}/{\linebreak[0]}World{\linebreak[0]}'}}.

\item {} 
Lists can be sorted in-place: \texttt{words{\linebreak[0]}.{\linebreak[0]}sort{\linebreak[0]}({\linebreak[0]})}.  To produce a separate,
sorted copy, use: \texttt{sorted{\linebreak[0]}({\linebreak[0]}words{\linebreak[0]})}.

\item {} 
We process each item in a string or list using a \texttt{\pysrckeyword{for}} statement:
\texttt{\pysrckeyword{for} {\linebreak[0]}word{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}phrase}.  This must be followed by the colon character
and an indented block of code, to be executed each time through the loop.

\item {} 
We test a condition using an \texttt{\pysrckeyword{if}} statement: \texttt{\pysrckeyword{if} {\linebreak[0]}len{\linebreak[0]}({\linebreak[0]}word{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}<{\linebreak[0]} {\linebreak[0]}5}.
This must be followed by the colon character and an indented block of
code, to be executed only if the condition is true.

\item {} 
A dictionary is used to map between arbitrary types of information,
such as a string and a number: \texttt{freq{\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}cat{\linebreak[0]}'}{]}{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}12}.  We create
dictionaries using the brace notation: \texttt{pos{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}{\{}{\linebreak[0]}{\}}},
\texttt{pos{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}{\{}\pysrcstring{'{\linebreak[0]}furiously{\linebreak[0]}'}:{\linebreak[0]} \pysrcstring{'{\linebreak[0]}adv{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}ideas{\linebreak[0]}'}:{\linebreak[0]} \pysrcstring{'{\linebreak[0]}n{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}colorless{\linebreak[0]}'}:{\linebreak[0]} \pysrcstring{'{\linebreak[0]}adj{\linebreak[0]}'}{\}}}.

\item {} 
Some functions are not available by default, but must be accessed using
Python's \texttt{\pysrckeyword{import}} statement.

\item {} 
Regular expressions are a powerful and flexible method of specifying
patterns. Once we have imported the \texttt{re} module, we can use
\texttt{re{\linebreak[0]}.{\linebreak[0]}findall{\linebreak[0]}({\linebreak[0]})} to find all substrings in a string that match a pattern,
and we can use \texttt{re{\linebreak[0]}.{\linebreak[0]}sub{\linebreak[0]}({\linebreak[0]})} to replace substrings of one sort with another.

\end{itemize}


%___________________________________________________________________________

\section{Further Reading}
\label{further-reading}


%___________________________________________________________________________

\subsection{Python}
\label{python}

Two freely available online texts are the following:
\begin{itemize}
\item {} 
Josh Cogliati, \emph{Non-Programmer's Tutorial for Python},
\href{http://en.wikibooks.org/wiki/Non-Programmer's_Tutorial_for_Python/Contents}{http://en.wikibooks.org/wiki/Non-Programmer's{\_}Tutorial{\_}for{\_}Python/Contents}

\item {} 
Allen B. Downey, Jeffrey Elkner and Chris Meyers,
\emph{How to Think Like a Computer Scientist: Learning with Python},
\href{http://www.ibiblio.org/obp/thinkCSpy/}{http://www.ibiblio.org/obp/thinkCSpy/}

\end{itemize}

\emph{An Introduction to Python}
\cite{vanrossum2006ip} is a Python tutorial by Guido van
Rossum, the inventor of Python and Fred L. Drake, Jr., the official
editor of the Python documentation. It is available online at
\href{http://docs.python.org/tut/tut.html}{http://docs.python.org/tut/tut.html}. A more detailed but still
introductory text is \cite{lutz2003lp}, which covers the essential
features of Python, and also provides an overview of the standard libraries.
% A more advanced text, [vanRossum2006IPLR]_ is the official reference
% for the Python language itself, and describes the syntax of Python and
% its built-in datatypes in depth. It is also available online at
% http://docs.python.org/ref/ref.html. 

\cite{beazley2006per} is a succinct reference book; although not suitable
as an introduction to Python, it is an excellent resource for
intermediate and advanced programmers.

Finally, it is always worth checking the official \emph{Python
Documentation} at \href{http://docs.python.org/}{http://docs.python.org/}.


%___________________________________________________________________________

\subsection{Regular Expressions}
\label{id10}

There are many references for regular expressions, both practical and
theoretical.  \cite{friedl2002mre} is a comprehensive and detailed manual
in using regular expressions, covering their syntax in most major
programming languages, including Python.

For an introductory
tutorial to using regular expressions in Python with the \texttt{re}
module, see A. M. Kuchling, \emph{Regular Expression HOWTO},
\href{http://www.amk.ca/python/howto/regex/}{http://www.amk.ca/python/howto/regex/}.

Chapter 3 of \cite{mertz2003tpp} provides a more extended tutorial on
Python's facilities for text processing with regular expressions.

\href{http://www.regular-expressions.info/}{http://www.regular-expressions.info/} is a useful online resource,
providing a tutorial and references to tools and other sources of
information.
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
\hypertarget{chap-words}{}\setcounter{chapter}{1}

%___________________________________________________________________________

\chapter{Words: The Building Blocks of Language}
\label{words-the-building-blocks-of-language}
\label{chap-words}
% TODO: cleaning web text 
% TODO: more on regular expressions, including () 
% TODO: add some literature references (esp to other intro linguistics textbooks) 
% TODO: talk about fact that English lexicon is open set (e.g. malware = malicious software) 
% TODO: add section on sentence tokenization (or sentence segmentation), and punkt 
% TODO: mention .words() methods for corpus readers for multiple languages 
% TODO: other issues
% - nltk.corpus.brown.items returns a tuple, not a list (cf discussion in ch 6)
% - invocation of pprint.pprint is a little clunky
% - regexp_tokenize() doesn't work when it is given a compiled pattern 


%___________________________________________________________________________

\section{Introduction}
\label{introduction}

{\noindent}
Language can be divided up into pieces of varying sizes, ranging from
morphemes to paragraphs.  In this chapter we will focus on words, the
most fundamental level for NLP.  Just what are words, and
how should we represent them in a machine?  These questions may seem
trivial, but we'll see that there are some important
issues involved in defining and representing words.
Once we've tackled them, we're in a good position to do further
processing, such as find related words and analyze the
style of a text (this chapter), to categorize words (\href{\#chap-tag}{Chapter 3}),
to group them into phrases (\href{\#chap-chunk}{Chapter 6} and Part II),
and to do a variety of data-intensive language processing tasks
(\href{\#chap-data-intensive}{Chapter 4}).

In the following sections, we will explore the division of text into
words; the distinction between types and tokens; sources of text data
including files, the web, and linguistic corpora; accessing these
sources using Python and NLTK; stemming and normalization;
the WordNet lexical database; and a variety of useful programming tasks involving words.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

From this chapter onwards, our program samples will assume you
begin your interactive session or your program with: \texttt{\pysrckeyword{import} {\linebreak[0]}nltk{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}re{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}pprint}
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\section{Tokens, Types and Texts}
\label{tokens-types-and-texts}

In Chapter chap-programming, we showed how a string could be split into a
list of words.  Once we have derived a list, the \texttt{len{\linebreak[0]}({\linebreak[0]})} function
will count the number of words it contains:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sentence = \pysrcstring{"This is the time -{}- and this is the record of the time."}
\pysrcprompt{>{}>{}> }words = sentence.split()
\pysrcprompt{>{}>{}> }len(words)
\pysrcoutput{13}}\end{alltt}

{\noindent} This process of segmenting a string of characters into words
is known as \index{tokenization}\textbf{tokenization}. Tokenization is a prelude to pretty
much everything else we might want to do in NLP, since it tells our
processing software what our basic units are. We will discuss
tokenization in more detail shortly.

We also pointed out that we could compile a list of the unique
vocabulary items in a string by using \texttt{set{\linebreak[0]}({\linebreak[0]})} to eliminate
duplicates:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }len(set(words))
\pysrcoutput{10}}\end{alltt}

{\noindent} So if we ask how many words there are in \texttt{sentence}, we get
different answers depending on whether we count duplicates.
Clearly we are using different senses of ``word'' here. To help
distinguish between them, let's introduce two terms:
\index{token}\textbf{token} and \index{type}\textbf{type}.  A word token is an individual occurrence
of a word in a concrete context; it exists in time and space.  A word
\index{type}\textbf{type} is a more abstract; it's what we're talking about when we
say that the three occurrences of \texttt{the} in \texttt{sentence} are  ``the same word.''

Something similar to a type-token distinction is reflected in the following
snippet of Python:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }words{[}2{]}
\pysrcoutput{'the'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }words{[}2{]} == words{[}8{]}
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }words{[}2{]} \pysrckeyword{is} words{[}8{]}
\pysrcoutput{False}
\pysrcoutput{}\pysrcprompt{>{}>{}> }words{[}2{]} \pysrckeyword{is} words{[}2{]}
\pysrcoutput{True}}\end{alltt}

{\noindent} The operator \texttt{={\linebreak[0]}=} tests whether two expressions are equal, and in
this case, it is testing for string-identity. This is the notion of
identity that was assumed by our use of \texttt{set{\linebreak[0]}({\linebreak[0]})} above. By contrast, the
\texttt{\pysrckeyword{is}} operator tests whether two objects are stored in the same
location of memory, and is therefore analogous to token-identity.
When we used \texttt{split{\linebreak[0]}({\linebreak[0]})} to turn a string into a list of words,
our tokenization method was to say that any strings that
are delimited by whitespace count as a word token. But this simple
approach doesn't always give the desired results.
Also, testing string-identity isn't a very useful criterion for
assigning tokens to types. We therefore need to address two questions
in more detail:
\emph{Tokenization:} Which substrings of the original text should be treated as word tokens?
\emph{Type definition:} How do we decide whether two tokens have the same type?

To see the problems with our first stab at defining tokens and types
in \texttt{sentence}, let's look at the actual tokens we found:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }set(words)
\pysrcoutput{set({[}'and', 'this', 'record', 'This', 'of', 'is', '-{}-', 'time.', 'time', 'the'{]})}}\end{alltt}

{\noindent}
Observe that \texttt{\pysrcstring{'{\linebreak[0]}time{\linebreak[0]}'}} and \texttt{\pysrcstring{'{\linebreak[0]}time{\linebreak[0]}.{\linebreak[0]}'}} are incorrectly
treated as distinct types since the trailing period
has been bundled with the rest of the word.
Although \texttt{\pysrcstring{'{\linebreak[0]}-{\linebreak[0]}-{\linebreak[0]}'}} is some kind of token, it's not a \docutilsroleemphasis{word} token.
Additionally, \texttt{\pysrcstring{'{\linebreak[0]}This{\linebreak[0]}'}} and \texttt{\pysrcstring{'{\linebreak[0]}this{\linebreak[0]}'}} are incorrectly distinguished
from each other, because of a difference in capitalization that should
be ignored.

If we turn to languages other than English, tokenizing text is even more
challenging.  In Chinese text there is no visual representation of word
boundaries.  Consider the following
three-character string: 爱国人 (in pinyin plus tones:
ai4 ``love'' (verb), guo3 ``country'', ren2 ``person''). This could
either be segmented as {[}爱国{]}人,
``country-loving person'' or as 爱{[}国人{]},
``love country-person.''

The terms \emph{token} and \emph{type} can also be applied to other linguistic
entities.  For example, a \index{sentence token}\textbf{sentence token} is an individual
occurrence of a sentence; but a \index{sentence type}\textbf{sentence type} is an abstract
sentence, without context. If I say the same sentence twice, I have
uttered two sentence tokens but only used one sentence type.  When the kind
of token or type is obvious from context, we will simply use the terms
token and type.

To summarize, we cannot just say that two word tokens have the same type if they are
the same string of characters.  We need to
consider a variety of factors in determining what counts as
the same word, and we need to be careful in how we
identify tokens in the first place.

Up till now, we have relied on getting our source texts by defining a string in
a fragment of Python code. However, this is impractical
for all but the simplest of texts, and makes it hard to present
realistic examples. So how do we get larger chunks of
text into our programs?  In the rest of this section, we will see how
to extract text from files, from the web, and from the corpora
distributed with NLTK.
\hypertarget{reading-files}{}

%___________________________________________________________________________

\subsection{Extracting Text from Files}
\label{extracting-text-from-files}
\label{reading-files}
% Monkey-patching to fake the file/web examples in this section:
% 
% >>> from StringIO import StringIO
% >>> def fake_open(filename, mode=None):
% ...     return StringIO('Hello World!\nThis is a test file.\n')
% >>> def fake_urlopen(url):
% ...     return StringIO('<!doctype html public "-//W3C//DTD HTML 4.0 Transitional//EN"')
% >>> open = fake_open
% >>> import urllib
% >>> urllib.urlopen = fake_urlopen 

It is easy to access local files in Python.  As an exercise, create a
file called \texttt{corpus{\linebreak[0]}.{\linebreak[0]}txt} using a text editor, and enter the
following text:
\begin{quote}{\ttfamily \raggedright \noindent
\small
Hello~World!~\\
This~is~a~test~file.
}\end{quote}

{\noindent}
Be sure to save the file as plain text. You also need to make sure
that you have saved the file in the same directory or folder in which
you are running the Python interactive interpreter.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

If you are using IDLE, you can easily create this file by
selecting the \emph{New Window} command in the \emph{File} menu, typing
the required text into this window, and then saving the file as
\texttt{corpus{\linebreak[0]}.{\linebreak[0]}txt} in the first directory that IDLE offers in the
pop-up dialogue box.
\end{boxedminipage}
\end{sffamily}
\end{center}

{\noindent}
The next step is to \index{open}\textbf{open} a file using the built-in function \texttt{open{\linebreak[0]}({\linebreak[0]})}
which takes two arguments, the name of the file, here \texttt{corpus{\linebreak[0]}.{\linebreak[0]}txt}, and the
mode to open the file with (\texttt{\pysrcstring{'{\linebreak[0]}r{\linebreak[0]}'}} means to open the file for reading,
and \texttt{\pysrcstring{'{\linebreak[0]}U{\linebreak[0]}'}} stands for ``Universal'', which lets us ignore the different
conventions used for marking newlines).
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }f = open(\pysrcstring{'corpus.txt'}, \pysrcstring{'rU'})}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

If the interpreter cannot find your file, it will give an
error like this:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }f = open(\pysrcstring{'corpus.txt'}, \pysrcstring{'rU'})
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{    File "<pyshell{\#}7>", line 1, in -toplevel-}
\pysrcexcept{    f = open('corpus.txt', 'rU')}
\pysrcexcept{IOError: {[}Errno 2{]} No such file or directory: 'corpus.txt'}}\end{alltt}

To check that the file that you are trying to open is really in the
right directory, use IDLE's \emph{Open} command in the \emph{File} menu;
this will display a list of all the files in the directory where
IDLE is running. An alternative is to examine the current
directory from within Python:
\end{boxedminipage}
\end{sffamily}
\end{center}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} os
\pysrcprompt{>{}>{}> }os.listdir(\pysrcstring{'.'})}\end{alltt}

{\noindent}
There are several methods for reading the file.
The following uses the method \texttt{read{\linebreak[0]}({\linebreak[0]})} on the file object
\texttt{f}; this reads the entire contents of a file into a string.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }f.read()
\pysrcoutput{'Hello World!{\textbackslash}nThis is a test file.{\textbackslash}n'}}\end{alltt}

{\noindent}
Recall that the \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}n{\linebreak[0]}'}} characters are \index{newlines}\textbf{newlines}; this
is equivalent to pressing \emph{Enter} on a keyboard and starting a new line.
Note that we can open and read a file in one step:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text = open(\pysrcstring{'corpus.txt'}, \pysrcstring{'rU'}).read()}\end{alltt}

{\noindent}
We can also read a file one line at a time using the \texttt{\pysrckeyword{for}} loop construct:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }f = open(\pysrcstring{'corpus.txt'}, \pysrcstring{'rU'})
\pysrcprompt{>{}>{}> }\pysrckeyword{for} line \pysrckeyword{in} f:
\pysrcmore{... }    \pysrckeyword{print} line{[}:-1{]}
\pysrcoutput{Hello world!}
\pysrcoutput{This is a test file.}}\end{alltt}

{\noindent}
Here we use the slice \texttt{{[}{\linebreak[0]}:{\linebreak[0]}-{\linebreak[0]}1{\linebreak[0]}{]}} to remove the newline character at the end of
the input line.


%___________________________________________________________________________

\subsection{Extracting Text from the Web}
\label{extracting-text-from-the-web}

Opening a web page is not much different to opening a file, except that
we use \texttt{urlopen{\linebreak[0]}({\linebreak[0]})}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} urllib \pysrckeyword{import} urlopen
\pysrcprompt{>{}>{}> }page = urlopen(\pysrcstring{"http://news.bbc.co.uk/"}).read()
\pysrcprompt{>{}>{}> }\pysrckeyword{print} page{[}:60{]}
\pysrcoutput{<!doctype html public "-//W3C//DTD HTML 4.0 Transitional//EN}}\end{alltt}

{\noindent}
Web pages are usually in HTML format.  To extract the text, we
need to strip out the HTML markup, i.e. remove all material enclosed in
angle brackets. Let's digress briefly to consider how to carry out
this task using regular expressions. Our first attempt might look as
follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }line = \pysrcstring{'<title>BBC NEWS | News Front Page</title>'}
\pysrcprompt{>{}>{}> }new = re.sub(r\pysrcstring{'<.*>'}, \pysrcstring{'{}'}, line)}\end{alltt}

{\noindent}
So the regular expression \texttt{\pysrcstring{'{\linebreak[0]}<{\linebreak[0]}.{\linebreak[0]}*{\linebreak[0]}>{\linebreak[0]}'}} is intended to match a
pair of left and right angle brackets, with a string of any characters
intervening. However, look at what the result is:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }new
\pysrcoutput{'{}'}}\end{alltt}

{\noindent}
What has happened here? The problem is twofold. First,
the wildcard \texttt{\pysrcstring{'{\linebreak[0]}.{\linebreak[0]}'}} matches any character other than \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}n{\linebreak[0]}'}},
so it will match \texttt{\pysrcstring{'{\linebreak[0]}>{\linebreak[0]}'}} and \texttt{\pysrcstring{'{\linebreak[0]}<{\linebreak[0]}'}}.
Second, the \texttt{\pysrcstring{'{\linebreak[0]}*{\linebreak[0]}'}} operator is ``greedy'', in the sense that it matches
as many characters as it can. In the above example,
\texttt{\pysrcstring{'{\linebreak[0]}.{\linebreak[0]}*{\linebreak[0]}'}} will return not the shortest match, namely
\texttt{\pysrcstring{'{\linebreak[0]}title{\linebreak[0]}'}}, but the longest match, \texttt{\pysrcstring{'{\linebreak[0]}title{\linebreak[0]}>{\linebreak[0]}BBC{\linebreak[0]} {\linebreak[0]}NEWS{\linebreak[0]} {\linebreak[0]}|{\linebreak[0]} {\linebreak[0]}News{\linebreak[0]} {\linebreak[0]}Front}
\pysrcstring{Page{\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}title{\linebreak[0]}'}}.   To get the \emph{shortest} match we have to use the \texttt{\pysrcstring{'{\linebreak[0]}*{\linebreak[0]}?{\linebreak[0]}'}} operator.
We will also normalize whitespace, replacing any sequence
of spaces, tabs or newlines (\texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}s{\linebreak[0]}+{\linebreak[0]}'}}) with a single space character.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }page = re.sub(\pysrcstring{'<.*?>'}, \pysrcstring{'{}'}, page)
\pysrcprompt{>{}>{}> }page = re.sub(\pysrcstring{'{\textbackslash}s+'}, \pysrcstring{' '}, page)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} page{[}:60{]}
\pysrcoutput{ BBC NEWS | News Front Page News Sport Weather World Service}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Note that your output for the above code may differ from ours,
because the BBC home page may have been changed since this
example was created.
\end{boxedminipage}
\end{sffamily}
\end{center}
% TODO: fix above example?  (make it current; issue with script on BBC site) 

You will probably find it useful to borrow the structure of the above code
snippet for future tasks involving regular expressions: each time
through a series of substitutions, the result of operating on \texttt{page}
gets assigned as the new value of \texttt{page}. This approach allows us to
decompose the transformations we need into a series of simple regular
expression substitutions, each of which can be tested and debugged on its own.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Getting text out of HTML is a sufficiently common task that NLTK provides
a helper function \texttt{nltk{\linebreak[0]}.{\linebreak[0]}clean{\_}html{\linebreak[0]}({\linebreak[0]})}, which takes an HTML string and returns text.
\end{boxedminipage}
\end{sffamily}
\end{center}
\hypertarget{sec-extracting-text-from-corpora}{}

%___________________________________________________________________________

\subsection{Extracting Text from NLTK Corpora}
\label{extracting-text-from-nltk-corpora}
\label{sec-extracting-text-from-corpora}

NLTK is distributed with several corpora and corpus samples and many
are supported by the \texttt{corpus} package.  Here we use
a selection of texts from the \href{http://www.gutenberg.org/}{Project Gutenberg} electronic text archive, and list the
files it contains:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.corpus.gutenberg.files()
\pysrcoutput{('austen-emma.txt', 'austen-persuasion.txt', 'austen-sense.txt', 'bible-kjv.txt',}
\pysrcoutput{'blake-poems.txt', 'blake-songs.txt', 'chesterton-ball.txt', 'chesterton-brown.txt',}
\pysrcoutput{'chesterton-thursday.txt', 'milton-paradise.txt', 'shakespeare-caesar.txt',}
\pysrcoutput{'shakespeare-hamlet.txt', 'shakespeare-macbeth.txt', 'whitman-leaves.txt')}}\end{alltt}

{\noindent}
We can count the number of tokens for each text in our Gutenberg sample
as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} book \pysrckeyword{in} nltk.corpus.gutenberg.files():
\pysrcmore{... }    \pysrckeyword{print} book + \pysrcstring{':'}, len(nltk.corpus.gutenberg.words(book))
\pysrcoutput{austen-emma.txt: 192432}
\pysrcoutput{austen-persuasion.txt: 98191}
\pysrcoutput{austen-sense.txt: 141586}
\pysrcoutput{bible-kjv.txt: 1010735}
\pysrcoutput{blake-poems.txt: 8360}
\pysrcoutput{blake-songs.txt: 6849}
\pysrcoutput{chesterton-ball.txt: 97396}
\pysrcoutput{chesterton-brown.txt: 89090}
\pysrcoutput{chesterton-thursday.txt: 69443}
\pysrcoutput{milton-paradise.txt: 97400}
\pysrcoutput{shakespeare-caesar.txt: 26687}
\pysrcoutput{shakespeare-hamlet.txt: 38212}
\pysrcoutput{shakespeare-macbeth.txt: 23992}
\pysrcoutput{whitman-leaves.txt: 154898}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

It is possible to use the methods described in \href{\#reading-files}{section 2.2.1}
along with \texttt{nltk{\linebreak[0]}.{\linebreak[0]}data{\linebreak[0]}.{\linebreak[0]}find{\linebreak[0]}({\linebreak[0]})} method to access and read the corpus files
directly.  The method described in this section is superior since it takes
care of tokenization and conveniently skips over the Gutenberg file header.
\end{boxedminipage}
\end{sffamily}
\end{center}

But note that this has several disadvantages.  The ones that come to
mind immediately are: (i) The corpus reader automatically strips out
the Gutenberg header; this version doesn't.  (ii) The corpus reader
uses a somewhat smarter method to break lines into words; this version
just splits on whitespace.  (iii) Using the corpus reader, you can
also access the documents by sentence or paragraph; doing that by
hand, you'd need to do some extra work.

The Brown Corpus was the first million-word,
part-of-speech tagged electronic corpus of English, created in 1961 at
Brown University.  Each of the sections \texttt{a} through \texttt{r} represents
a different genre, as shown in \href{\#brown-categories}{Table 2.1}.
\hypertarget{brown-categories}{}
\leavevmode
\begin{longtable}[c]{|p{0.051\locallinewidth}|p{0.203\locallinewidth}|p{0.051\locallinewidth}|p{0.226\locallinewidth}|p{0.051\locallinewidth}|p{0.203\locallinewidth}|}
\hline
\textbf{
Sec
} & \textbf{
Genre
} & \textbf{
Sec
} & \textbf{
Genre
} & \textbf{
Sec
} & \textbf{
Genre
} \\
\hline
\endhead

a
 & 
Press: Reportage
 & 
b
 & 
Press: Editorial
 & 
c
 & 
Press: Reviews
 \\
\hline

d
 & 
Religion
 & 
e
 & 
Skill and Hobbies
 & 
f
 & 
Popular Lore
 \\
\hline

g
 & 
Belles-Lettres
 & 
h
 & 
Government
 & 
j
 & 
Learned
 \\
\hline

k
 & 
Fiction: General
 & 
k
 & 
Fiction: General
 & 
l
 & 
Fiction: Mystery
 \\
\hline

m
 & 
Fiction: Science
 & 
n
 & 
Fiction: Adventure
 & 
p
 & 
Fiction: Romance
 \\
\hline

r
 & 
Humor
 &  &  &  &  \\
\hline
\caption{
Sections of the Brown Corpus
}\end{longtable}

We can access the corpus as a list of words, or a list of sentences (where each sentence
is itself just a list of words).  We can optionally specify a section of the corpus to
read:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.corpus.brown.categories()
\pysrcoutput{{[}'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', 'p', 'r'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.brown.words(categories=\pysrcstring{'a'})
\pysrcoutput{{[}'The', 'Fulton', 'County', 'Grand', 'Jury', 'said', ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.brown.sents(categories=\pysrcstring{'a'})
\pysrcoutput{{[}{[}'The', 'Fulton', 'County'...{]}, {[}'The', 'jury', 'further'...{]}, ...{]}}}\end{alltt}

NLTK comes with corpora for many languages, though in some cases
you will need to learn how to manipulate character encodings in Python
before using these corpora.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.corpus.nps{\_}chat.words()
\pysrcoutput{{[}'now', 'im', 'left', 'with', 'this', 'gay', 'name', ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.cess{\_}esp.words()
\pysrcoutput{{[}'El', 'grupo', 'estatal', 'Electricit{\textbackslash}xe9{\_}de{\_}France', ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.floresta.words()
\pysrcoutput{{[}'Um', 'revivalismo', 'refrescante', 'O', '7{\_}e{\_}Meio', ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.udhr.words(\pysrcstring{'Javanese-Latin1'}){[}11:{]}
\pysrcoutput{{[}'Saben', 'umat', 'manungsa', 'lair', 'kanthi', 'hak', ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.indian.words(\pysrcstring{'hindi.pos'})
\pysrcoutput{{[}'{\textbackslash}xe0{\textbackslash}xa4{\textbackslash}xaa{\textbackslash}xe0{\textbackslash}xa5{\textbackslash}x82{\textbackslash}xe0{\textbackslash}xa4{\textbackslash}xb0{\textbackslash}xe0{\textbackslash}xa5{\textbackslash}x8d{\textbackslash}xe0{\textbackslash}xa4{\textbackslash}xa3',}
\pysrcoutput{'{\textbackslash}xe0{\textbackslash}xa4{\textbackslash}xaa{\textbackslash}xe0{\textbackslash}xa5{\textbackslash}x8d{\textbackslash}xe0{\textbackslash}xa4{\textbackslash}xb0{\textbackslash}xe0{\textbackslash}xa4{\textbackslash}xa4{\textbackslash}xe0{\textbackslash}xa4{\textbackslash}xbf{\textbackslash}xe0{\textbackslash}xa4{\textbackslash}xac{\textbackslash}xe0{\textbackslash}xa4{\textbackslash}x82{\textbackslash}xe0{\textbackslash}xa4{\textbackslash}xa7', ...{]}}}\end{alltt}

Before concluding this section, we return to the original topic of
distinguishing tokens and types.  Now that we can access substantial
quantities of text, we will give a preview of the interesting computations
we will be learning how to do (without yet explaining all the details).
\href{\#vocabulary-growth}{Listing 2.1} computes vocabulary growth curves
for US Presidents, shown in \href{\#fig-vocabulary-growth}{Figure 2.1}
(a color figure in the online version).
These curves show the number of word types seen after \docutilsrolemath{n} word tokens
have been read.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

\href{\#vocabulary-growth}{Listing 2.1} uses the PyLab package which supports sophisticated
plotting functions with a MATLAB-style interface.  For more information about
this package please see \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}matplotlib{\linebreak[0]}.{\linebreak[0]}sourceforge{\linebreak[0]}.{\linebreak[0]}net{\linebreak[0]}/}.  The listing also
uses the \texttt{yield} statement, which will be explained in \href{\#chap-structured-programming}{Chapter 5}.
\end{boxedminipage}
\end{sffamily}
\end{center}
\hypertarget{vocabulary-growth}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{vocab{\_}growth}(texts):
    vocabulary = set()
    \pysrckeyword{for} text \pysrckeyword{in} texts:
        \pysrckeyword{for} word \pysrckeyword{in} text:
            vocabulary.add(word)
            yield len(vocabulary)

\pysrckeyword{def} \pysrcdefname{speeches}():
    presidents = {[}{]}
    texts = nltk.defaultdict(list)
    \pysrckeyword{for} speech \pysrckeyword{in} nltk.corpus.state{\_}union.files():
        president = speech.split(\pysrcstring{'-'}){[}1{]}
        \pysrckeyword{if} president \pysrckeyword{not} \pysrckeyword{in} texts:
            presidents.append(president)
        texts{[}president{]}.append(nltk.corpus.state{\_}union.words(speech))
    return {[}(president, texts{[}president{]}) \pysrckeyword{for} president \pysrckeyword{in} presidents{]}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} pylab
\pysrcprompt{>{}>{}> }\pysrckeyword{for} president, texts \pysrckeyword{in} speeches(){[}-7:{]}:
\pysrcmore{... }    growth = list(vocab{\_}growth(texts)){[}:10000{]}
\pysrcmore{... }    pylab.plot(growth, label=president, linewidth=2)
\pysrcprompt{>{}>{}> }pylab.title(\pysrcstring{'Vocabulary Growth in State-of-the-Union Addresses'})
\pysrcprompt{>{}>{}> }pylab.legend(loc=\pysrcstring{'lower right'})
\pysrcprompt{>{}>{}> }pylab.show()         }\end{alltt}
\caption{Vocabulary Growth in State-of-the-Union Addresses}
\vspace{1ex}
\hrule
\end{pylisting}
\hypertarget{fig-vocabulary-growth}{}\begin{figure}[htbp]
\centering

\scalebox{0.128571}{\includegraphics{../images/vocabulary-growth.png}}
\caption{Vocabulary Growth in State-of-the-Union Addresses}\end{figure}


%___________________________________________________________________________

\subsection{Exercises}
\label{id13}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Create a small text file, and write a program to read it and print it
with a line number at the start of each line.  (Make sure you don't introduce
an extra blank line between each line.)

\item {} 
☼ Use the corpus module to read \texttt{austen{\linebreak[0]}-{\linebreak[0]}persuasion{\linebreak[0]}.{\linebreak[0]}txt}.
How many word tokens does this book have?  How many word types?

\item {} 
☼ Use the Brown corpus reader \texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}brown{\linebreak[0]}.{\linebreak[0]}words{\linebreak[0]}({\linebreak[0]})} or the Web text corpus
reader \texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}webtext{\linebreak[0]}.{\linebreak[0]}words{\linebreak[0]}({\linebreak[0]})} to access some sample text in two different genres.

\item {} 
☼ Use the Brown corpus reader \texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}brown{\linebreak[0]}.{\linebreak[0]}sents{\linebreak[0]}({\linebreak[0]})} to find sentence-initial
examples of the word \docutilsroleexample{however}.  Check whether these conform to Strunk and White's
prohibition against sentence-initial \docutilsroleexample{however} used to mean ``although''.

\item {} 
☼ Read in the texts of the \emph{State of the Union} addresses, using the
\texttt{state{\_}union} corpus reader.  Count occurrences of \texttt{men}, \texttt{women},
and \texttt{people} in each document.  What has happened to the usage of these
words over time?

\item {} 
◑ Write code to read a file and print the lines in reverse order,
so that the last line is listed first.

\item {} 
◑ Read in some text from a corpus, tokenize it, and print the list of
all \docutilsroleexample{wh}-word types that occur.  (\docutilsroleexample{wh}-words in English
are used in questions, relative clauses and exclamations:
\docutilsroleexample{who}, \docutilsroleexample{which}, \docutilsroleexample{what}, and so on.) Print
them in order.  Are any words duplicated in this list, because of
the presence of case distinctions or punctuation?

\item {} 
◑ Write code to access a favorite webpage and extract some text from it.
For example, access a weather site and extract the forecast top
temperature for your town or city today.

\item {} 
◑ Write a function \texttt{unknown{\linebreak[0]}({\linebreak[0]})} that takes a URL as its argument,
and returns a list of unknown words that occur on that webpage.
In order to do this, extract all substrings consisting of lowercase letters
(using \texttt{re{\linebreak[0]}.{\linebreak[0]}findall{\linebreak[0]}({\linebreak[0]})}) and remove any items from this set that occur
in the words corpus (\texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}words}).  Try to categorize these words
manually and discuss your findings.

\item {} 
◑ Examine the results of processing the URL
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}news{\linebreak[0]}.{\linebreak[0]}bbc{\linebreak[0]}.{\linebreak[0]}co{\linebreak[0]}.{\linebreak[0]}uk{\linebreak[0]}/} using the regular expressions suggested
above. You will see that there is still a fair amount of
non-textual data there, particularly Javascript commands. You may
also find that sentence breaks have not been properly
preserved. Define further regular expressions that improve the
extraction of text from this web page.

\item {} 
◑ Take a copy of the \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}news{\linebreak[0]}.{\linebreak[0]}bbc{\linebreak[0]}.{\linebreak[0]}co{\linebreak[0]}.{\linebreak[0]}uk{\linebreak[0]}/} over three different
days, say at two-day intervals. This should give you three
different files, \texttt{bbc1{\linebreak[0]}.{\linebreak[0]}txt}, \texttt{bbc2{\linebreak[0]}.{\linebreak[0]}txt} and \texttt{bbc3{\linebreak[0]}.{\linebreak[0]}txt}, each
corresponding to a different snapshot of world events. Collect the
100 most frequent word tokens for each file. What can you tell
from the changes in frequency?

\item {} 
◑ Define a function \texttt{ghits{\linebreak[0]}({\linebreak[0]})} that takes a word as its argument and
builds a Google query string of the form \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}www{\linebreak[0]}.{\linebreak[0]}google{\linebreak[0]}.{\linebreak[0]}com{\linebreak[0]}/{\linebreak[0]}search{\linebreak[0]}?{\linebreak[0]}q{\linebreak[0]}={\linebreak[0]}word}.
Strip the HTML markup and normalize whitespace.  Search for a substring
of the form \texttt{Results{\linebreak[0]} {\linebreak[0]}1{\linebreak[0]} {\linebreak[0]}-{\linebreak[0]} {\linebreak[0]}10{\linebreak[0]} {\linebreak[0]}of{\linebreak[0]} {\linebreak[0]}about}, followed by some number
\docutilsrolemath{n},  and extract \docutilsrolemath{n}.
Convert this to an integer and return it.

\item {} 
◑ Try running the various chatbots included with NLTK, using
\texttt{nltk{\linebreak[0]}.{\linebreak[0]}chat{\linebreak[0]}.{\linebreak[0]}demo{\linebreak[0]}({\linebreak[0]})}.  How \emph{intelligent} are these programs?
Take a look at the program code
and see if you can discover how it works.  You can find the code
online at: \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}/{\linebreak[0]}chat{\linebreak[0]}/}.

\item {} 
★ Define a function \texttt{find{\_}language{\linebreak[0]}({\linebreak[0]})} that takes a string
as its argument, and returns a list of languages that have that
string as a word.  Use the \texttt{udhr} corpus and limit your searches
to files in the Latin-1 encoding.

\end{list}


%___________________________________________________________________________

\section{Tokenization and Normalization}
\label{tokenization-and-normalization}

Tokenization, as we saw, is the task of extracting a sequence of elementary tokens
that constitute a piece of language data. In our first attempt to
carry out this task, we started off with a string of characters, and
used the \texttt{split{\linebreak[0]}({\linebreak[0]})} method to break the string at whitespace
characters. Recall that ``whitespace'' covers not only inter-word space,
but also tabs and newlines.
We pointed out that tokenization
based solely on whitespace is too simplistic for most applications.
In this section we will take a more sophisticated approach, using
regular expressions to specify which character sequences should be
treated as words.  We will also look at ways to normalize tokens.


%___________________________________________________________________________

\subsection{Tokenization with Regular Expressions}
\label{tokenization-with-regular-expressions}

The function \texttt{nltk{\linebreak[0]}.{\linebreak[0]}tokenize{\linebreak[0]}.{\linebreak[0]}regexp{\_}tokenize{\linebreak[0]}({\linebreak[0]})} takes a text string and a
regular expression, and returns the list of substrings that match the
regular expression.  To define a tokenizer that includes punctuation
as separate tokens, we could do the following:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text = \pysrcstring{'{}'{}'Hello.  Isn't this fun?'{}'{}'}
\pysrcprompt{>{}>{}> }pattern = r\pysrcstring{'{\textbackslash}w+|{[}{\textasciicircum}{\textbackslash}w{\textbackslash}s{]}+'}
\pysrcprompt{>{}>{}> }nltk.tokenize.regexp{\_}tokenize(text, pattern)
\pysrcoutput{{[}'Hello', '.', 'Isn', "'", 't', 'this', 'fun', '?'{]}}}\end{alltt}

{\noindent}
The regular expression in this example will match a sequence
consisting of one or more word characters \texttt{{\textbackslash}{\linebreak[0]}w{\linebreak[0]}+}.  It will also match
a sequence consisting of one or more punctuation characters (or
non-word, non-space characters \texttt{{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]}{\textbackslash}{\linebreak[0]}w{\linebreak[0]}{\textbackslash}{\linebreak[0]}s{\linebreak[0]}{]}{\linebreak[0]}+}).  This is another negated range
expression; it matches one or more characters that are not word
characters (i.e., not a match for \texttt{{\textbackslash}{\linebreak[0]}w}) and not a whitespace
character (i.e., not a match for \texttt{{\textbackslash}{\linebreak[0]}s}).  We use the disjunction
operator \texttt{|} to combine these into a single complex expression
\texttt{{\textbackslash}{\linebreak[0]}w{\linebreak[0]}+{\linebreak[0]}|{\linebreak[0]}{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]}{\textbackslash}{\linebreak[0]}w{\linebreak[0]}{\textbackslash}{\linebreak[0]}s{\linebreak[0]}{]}{\linebreak[0]}+}.

There are a number of ways we could improve on this regular
expression.  For example, it currently breaks \docutilsroleexample{{\$}22.50} into four
tokens; we might want it to treat this as a single token.
Similarly, \titlereference{U.S.A.} should count as a single token.  We can
deal with these by adding further cases to the regular
expression.  For readability we will break it up and insert comments, and
insert the special \texttt{({\linebreak[0]}?{\linebreak[0]}x{\linebreak[0]})} ``verbose flag'' so that Python knows
to strip out the embedded whitespace and comments.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text = \pysrcstring{'That poster costs {\$}22.40.'}
\pysrcprompt{>{}>{}> }pattern = r\pysrcstring{'{}'{}'(?x)}
\pysrcmore{... }\pysrcstring{    {\textbackslash}w+               {\#} sequences of 'word' characters}
\pysrcmore{... }\pysrcstring{  | {\textbackslash}{\$}?{\textbackslash}d+({\textbackslash}.{\textbackslash}d+)?    {\#} currency amounts, e.g. {\$}12.50}
\pysrcmore{... }\pysrcstring{  | ({[}A-Z{]}{\textbackslash}.)+        {\#} abbreviations, e.g. U.S.A.}
\pysrcmore{... }\pysrcstring{  | {[}{\textasciicircum}{\textbackslash}w{\textbackslash}s{]}+          {\#} sequences of punctuation}
\pysrcmore{... }\pysrcstring{'{}'{}'}
\pysrcprompt{>{}>{}> }nltk.tokenize.regexp{\_}tokenize(text, pattern)
\pysrcoutput{{[}'That', 'poster', 'costs', '{\$}22.40', '.'{]}}}\end{alltt}

It is sometimes more convenient to write a regular expression
matching the material that appears \emph{between} tokens, such as whitespace
and punctuation.  The \texttt{nltk{\linebreak[0]}.{\linebreak[0]}tokenize{\linebreak[0]}.{\linebreak[0]}regexp{\_}tokenize{\linebreak[0]}({\linebreak[0]})} function permits
an optional boolean parameter \texttt{gaps}; when set to \texttt{True} the
pattern is matched against the gaps.  For example, we could define
a whitespace tokenizer as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.tokenize.regexp{\_}tokenize(text, pattern=r\pysrcstring{'{\textbackslash}s+'}, gaps=True)
\pysrcoutput{{[}'That', 'poster', 'costs', '{\$}22.40.'{]}}}\end{alltt}

It is more convenient to call NLTK's whitespace
tokenizer directly, as \texttt{nltk{\linebreak[0]}.{\linebreak[0]}WhitespaceTokenizer{\linebreak[0]}({\linebreak[0]}text{\linebreak[0]})}.
(However, in this case is generally better to use Python's \texttt{split{\linebreak[0]}({\linebreak[0]})}
method, defined on strings: \texttt{text{\linebreak[0]}.{\linebreak[0]}split{\linebreak[0]}({\linebreak[0]})}.)


%___________________________________________________________________________

\subsection{Lemmatization and Normalization}
\label{lemmatization-and-normalization}

Earlier we talked about counting word tokens, and completely ignored
the rest of the sentence in which these tokens appeared.  Thus, for an
example like \docutilsroleexample{I saw the saw}, we would have treated both \docutilsroleexample{saw}
tokens as instances of the same type.  However, one is a form of the
verb \docutilsroleexample{see}, and the other is the name of a cutting instrument.
How do we know that these two forms of \docutilsroleexample{saw} are unrelated?  One
answer is that as speakers of English, we know that these would appear
as different entries in a dictionary. Another, more empiricist, answer
is that if we looked at a large enough number of texts, it would
become clear that the two forms have very different distributions. For
example, only the noun \docutilsroleexample{saw} will occur immediately after
determiners such as \docutilsroleexample{the}.  Distinct words that have the same
written form are called \index{homographs}\textbf{homographs}. We can distinguish
homographs with the help of context; often the previous word suffices.
We will explore this idea of context briefly, before addressing the
main topic of this section.

As a first approximation to discovering the distribution of a word, we
can look at all the bigrams it occurs in.
A \index{bigram}\textbf{bigram} is simply a pair of words.
For example, in the sentence \docutilsroleexample{She sells sea shells by the sea shore},
the bigrams are
\docutilsroleexample{She sells},
\docutilsroleexample{sells sea},
\docutilsroleexample{sea shells},
\docutilsroleexample{shells by},
\docutilsroleexample{by the},
\docutilsroleexample{the sea},
\docutilsroleexample{sea shore}.
Let's consider all bigrams from the Brown Corpus that have the
word \docutilsroleexample{often} as first element.
Here is a small selection, ordered by their counts:
\begin{quote}{\ttfamily \raggedright \noindent
\small
often~,~~~~~~~~~~~~~16~\\
often~a~~~~~~~~~~~~~10~\\
often~in~~~~~~~~~~~~8~\\
often~than~~~~~~~~~~7~\\
often~the~~~~~~~~~~~7~\\
often~been~~~~~~~~~~6~\\
often~do~~~~~~~~~~~~5~\\
often~called~~~~~~~~4~\\
often~appear~~~~~~~~3~\\
often~were~~~~~~~~~~3~\\
often~appeared~~~~~~2~\\
often~are~~~~~~~~~~~2~\\
often~did~~~~~~~~~~~2~\\
often~is~~~~~~~~~~~~2~\\
often~appears~~~~~~~1~\\
often~call~~~~~~~~~~1
}\end{quote}

{\noindent}
In the topmost entry, we see that \docutilsroleexample{often} is frequently followed
by a comma.  This suggests that \docutilsroleexample{often} is common at the end of
phrases. We also see that \docutilsroleexample{often} precedes verbs, presumably as an
adverbial modifier.  We might conclude that when \docutilsroleexample{saw}
appears in the context \docutilsroleexample{often saw}, then \docutilsroleexample{saw} is being used
as a verb.

You will also see that this list includes different grammatical forms
of the same verb. We can form separate groups consisting of
\docutilsroleexample{appear} {\textasciitilde} \docutilsroleexample{appears} {\textasciitilde} \docutilsroleexample{appeared}; \docutilsroleexample{call}
{\textasciitilde} \docutilsroleexample{called}; \docutilsroleexample{do} {\textasciitilde} \docutilsroleexample{did}; and \docutilsroleexample{been}
{\textasciitilde} \docutilsroleexample{were} {\textasciitilde} \docutilsroleexample{are} {\textasciitilde} \docutilsroleexample{is}.  It is common
in linguistics to say that two forms such as \docutilsroleexample{appear} and
\docutilsroleexample{appeared} belong to a more abstract notion of a word called a
\index{lexeme}\textbf{lexeme}; by contrast, \docutilsroleexample{appeared} and \docutilsroleexample{called} belong to
different lexemes. You can think of a lexeme as corresponding to an
entry in a dictionary, and a \index{lemma}\textbf{lemma} as the headword for that entry.
By convention, small capitals are used when referring to a lexeme or lemma: \docutilsrolelex{appear}.

Although \docutilsroleexample{appeared} and \docutilsroleexample{called} belong to different lexemes,
they do have something in common: they are both past tense forms. This
is signaled by the segment \docutilsroleexample{-ed}, which we call a morphological
\index{suffix}\textbf{suffix}. We also say that such morphologically complex forms are
\index{inflected}\textbf{inflected}.  If we strip off the suffix, we get something called
the \index{stem}\textbf{stem}, namely \docutilsroleexample{appear} and \docutilsroleexample{call} respectively. While
\docutilsroleexample{appeared}, \docutilsroleexample{appears} and \docutilsroleexample{appearing} are all
morphologically inflected, \docutilsroleexample{appear} lacks any morphological
inflection and is therefore termed the \index{base}\textbf{base} form. In English,
the base form is conventionally used as the \index{lemma}\textbf{lemma} for a word.

Our notion of context would be more compact if we could group
different forms of the various verbs into their lemmas; then we could
study which verb lexemes are typically modified by a particular
adverb. \index{Lemmatization}\textbf{Lemmatization} — the process of mapping words
to their lemmas — would yield the
following picture of the distribution of \docutilsroleexample{often}.
Here, the counts for \docutilsroleexample{often appear} (3),
\docutilsroleexample{often appeared} (2) and \docutilsroleexample{often appears} (1)
are combined into a single line.
\begin{quote}
\begin{quote}{\ttfamily \raggedright \noindent
\small
often~,~~~~~~~~~~~~~16~\\
often~a~~~~~~~~~~~~~10~\\
often~be~~~~~~~~~~~~13~\\
often~in~~~~~~~~~~~~8~\\
often~than~~~~~~~~~~7~\\
often~the~~~~~~~~~~~7~\\
often~do~~~~~~~~~~~~7~\\
often~appear~~~~~~~~6~\\
often~call~~~~~~~~~~5
}\end{quote}
\end{quote}

Lemmatization is a rather sophisticated process that uses rules
for the regular word patterns, and table look-up for
the irregular patterns. Within NLTK, we can use off-the-shelf
stemmers, such as the \index{Porter Stemmer}\textbf{Porter Stemmer}, the \index{Lancaster Stemmer}\textbf{Lancaster Stemmer},
and the stemmer that comes with WordNet, e.g.:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }stemmer = nltk.PorterStemmer()
\pysrcprompt{>{}>{}> }verbs = {[}\pysrcstring{'appears'}, \pysrcstring{'appear'}, \pysrcstring{'appeared'}, \pysrcstring{'calling'}, \pysrcstring{'called'}{]}
\pysrcprompt{>{}>{}> }stems = {[}{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} verb \pysrckeyword{in} verbs:
\pysrcmore{... }    stemmed{\_}verb = stemmer.stem(verb)
\pysrcmore{... }    stems.append(stemmed{\_}verb)
\pysrcprompt{>{}>{}> }sorted(set(stems))
\pysrcoutput{{[}'appear', 'call'{]}}}\end{alltt}

Stemmers for other languages are added to NLTK as they are contributed,
e.g. the RSLP Portuguese Stemmer, \texttt{nltk{\linebreak[0]}.{\linebreak[0]}RSLPStemmer{\linebreak[0]}({\linebreak[0]})}.

Lemmatization and stemming are special cases of \index{normalization}\textbf{normalization}.
They identify a canonical representative for a set of related word forms.
Normalization collapses distinctions.  Exactly how we normalize words depends
on the application.  Often, we convert everything into lower
case so that we can ignore the written distinction between sentence-initial
words and the rest of the words in the sentence.
The Python string method \texttt{lower{\linebreak[0]}({\linebreak[0]})} will accomplish this for us:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }str = \pysrcstring{'This is the time'}
\pysrcprompt{>{}>{}> }str.lower()
\pysrcoutput{'this is the time'}}\end{alltt}
% We need to be careful, however; case normalization will also collapse the `New`:lx:
% of `New York`:lx: with the `new`:lx: of `my new car`:lx:.
% [SB: this mixes the issue with named entity detection, and should probably
% be left to the engineering chapter] 

A final issue for normalization is the presence of contractions, such
as \docutilsroleexample{didn't}.  If we are analyzing the meaning
of a sentence, it would probably be more useful to normalize this
form to two separate forms: \docutilsroleexample{did} and \docutilsroleexample{n't} (or \docutilsroleexample{not}).
\hypertarget{sec-list-comprehensions}{}

%___________________________________________________________________________

\subsection{Transforming Lists}
\label{transforming-lists}
\label{sec-list-comprehensions}

Lemmatization and normalization involve applying the same operation to
each word token in a text.  \index{List comprehensions}\textbf{List comprehensions} are a convenient Python
construct for doing this.  Here we lowercase each word:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = {[}\pysrcstring{'The'}, \pysrcstring{'dog'}, \pysrcstring{'gave'}, \pysrcstring{'John'}, \pysrcstring{'the'}, \pysrcstring{'newspaper'}{]}
\pysrcprompt{>{}>{}> }{[}word.lower() \pysrckeyword{for} word \pysrckeyword{in} sent{]}
\pysrcoutput{{[}'the', 'dog', 'gave', 'john', 'the', 'newspaper'{]}}}\end{alltt}

{\noindent}
A list comprehension usually has the form
\texttt{{[}{\linebreak[0]}item{\linebreak[0]}.{\linebreak[0]}foo{\linebreak[0]}({\linebreak[0]}){\linebreak[0]} \pysrckeyword{for} {\linebreak[0]}item{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}sequence{\linebreak[0]}{]}},
or \texttt{{[}{\linebreak[0]}foo{\linebreak[0]}({\linebreak[0]}item{\linebreak[0]}){\linebreak[0]} \pysrckeyword{for} {\linebreak[0]}item{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}sequence{\linebreak[0]}{]}}.
It creates a list but applying an operation to every item in
the supplied sequence.
Here we rewrite the loop for identifying verb stems that we saw in
the previous section:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }{[}stemmer.stem(verb) \pysrckeyword{for} verb \pysrckeyword{in} verbs{]}
\pysrcoutput{{[}'appear', 'appear', 'appear', 'call', 'call'{]}}}\end{alltt}

Now we can eliminate repeats using \texttt{set{\linebreak[0]}({\linebreak[0]})}, by passing
the list comprehension as an argument.  We can actually
leave out the square brackets, as will be explained further
in \href{\#chap-applied-programming}{Chapter 9}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }set(stemmer.stem(verb) \pysrckeyword{for} verb \pysrckeyword{in} verbs)
\pysrcoutput{set({[}'call', 'appear'{]})}}\end{alltt}

{\noindent}
This syntax might be reminiscent of the notation used for building
sets, e.g. {\{}(x,y) {\textbar} x$^{2}$ + y$^{2}$ = 1{\}}.
(We will return to sets later in \href{\#sec-sets}{Section 9}).
Just as this set definition incorporates a constraint, list
comprehensions can constrain the items they include.  In the next
example we remove some non-content words from a list of words:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{is{\_}lexical}(word):
\pysrcmore{... }    return word.lower() \pysrckeyword{not} \pysrckeyword{in} (\pysrcstring{'a'}, \pysrcstring{'an'}, \pysrcstring{'the'}, \pysrcstring{'that'}, \pysrcstring{'to'})
\pysrcprompt{>{}>{}> }{[}word \pysrckeyword{for} word \pysrckeyword{in} sent \pysrckeyword{if} is{\_}lexical(word){]}
\pysrcoutput{{[}'dog', 'gave', 'John', 'newspaper'{]}}}\end{alltt}

{\noindent}
Now we can combine the two ideas (constraints and normalization),
to pull out the content words and normalize them.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }{[}word.lower() \pysrckeyword{for} word \pysrckeyword{in} sent \pysrckeyword{if} is{\_}lexical(word){]}
\pysrcoutput{{[}'dog', 'gave', 'john', 'newspaper'{]}}}\end{alltt}

List comprehensions can build nested structures too.  For example,
the following code builds a list of tuples, where each tuple consists
of a word and its stem.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = nltk.corpus.brown.sents(categories=\pysrcstring{'a'}){[}0{]}
\pysrcprompt{>{}>{}> }{[}(x, stemmer.stem(x).lower()) \pysrckeyword{for} x \pysrckeyword{in} sent{]}
\pysrcoutput{{[}('The', 'the'), ('Fulton', 'fulton'), ('County', 'counti'),}
\pysrcoutput{('Grand', 'grand'), ('Jury', 'juri'), ('said', 'said'), ('Friday', 'friday'),}
\pysrcoutput{('an', 'an'), ('investigation', 'investig'), ('of', 'of'),}
\pysrcoutput{("Atlanta's", "atlanta'"), ('recent', 'recent'), ('primary', 'primari'),}
\pysrcoutput{('election', 'elect'), ('produced', 'produc'), ('`{}`', '`{}`'), ('no', 'no'),}
\pysrcoutput{('evidence', 'evid'), ("'{}'", "'{}'"), ('that', 'that'), ('any', 'ani'),}
\pysrcoutput{('irregularities', 'irregular'), ('took', 'took'), ('place', 'place'), ('.', '.'){]}}}\end{alltt}
\hypertarget{sentence-segmentation}{}

%___________________________________________________________________________

\subsection{Sentence Segmentation}
\label{id14}
\label{sentence-segmentation}

Manipulating texts at the level of individual words often presupposes
the ability to divide a text into individual sentences.  As we have
seen, some corpora already provide access at the sentence level.  In
the following example, we compute the average number of words per
sentence in the Brown Corpus:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }len(nltk.corpus.brown.words()) / len(nltk.corpus.brown.sents())
\pysrcoutput{20}}\end{alltt}

In other cases, the text is only available as a stream of characters.  Before
doing word tokenization, we need to do sentence segmentation.  NLTK facilitates
this by including the Punkt sentence segmenter \cite{kissstrunk2006}, along with
supporting data for English.  Here is an example of its use in segmenting
the text of a novel:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent{\_}tokenizer=nltk.data.load(\pysrcstring{'tokenizers/punkt/english.pickle'})
\pysrcprompt{>{}>{}> }text = nltk.corpus.gutenberg.raw(\pysrcstring{'chesterton-thursday.txt'})
\pysrcprompt{>{}>{}> }sents = sent{\_}tokenizer.tokenize(text)
\pysrcprompt{>{}>{}> }pprint(sents{[}171:181{]})
\pysrcoutput{{[}'"Nonsense!',}
\pysrcoutput{ '" said Gregory, who was very rational when anyone else{\textbackslash}nattempted paradox.',}
\pysrcoutput{ '"Why do all the clerks and navvies in the{\textbackslash}nrailway trains look so sad and tired, so very sad and tired?',}
\pysrcoutput{ 'I will{\textbackslash}ntell you.',}
\pysrcoutput{ 'It is because they know that the train is going right.',}
\pysrcoutput{ 'It{\textbackslash}nis because they know that whatever place they have taken a ticket{\textbackslash}nfor that place they will reach.',}
\pysrcoutput{ 'It is because after they have{\textbackslash}npassed Sloane Square they know that the next station must be{\textbackslash}nVictoria, and nothing but Victoria.',}
\pysrcoutput{ 'Oh, their wild rapture!',}
\pysrcoutput{ 'oh,{\textbackslash}ntheir eyes like stars and their souls again in Eden, if the next{\textbackslash}nstation were unaccountably Baker Street!'}
\pysrcoutput{ '"{\textbackslash}n{\textbackslash}n"It is you who are unpoetical," replied the poet Syme.'{]}}}\end{alltt}

Notice that this example is really a single sentence, reporting the speech of Mr Lucian Gregory.
However, the quoted speech contains several sentences, and these have been split into individual
strings.  This is reasonable behavior for most applications.


%___________________________________________________________________________

\subsection{Exercises}
\label{id16}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ \textbf{Regular expression tokenizers:}
Save some text into a file \texttt{corpus{\linebreak[0]}.{\linebreak[0]}txt}.  Define a function \texttt{load{\linebreak[0]}({\linebreak[0]}f{\linebreak[0]})}
that reads from the file named in its sole argument, and returns a string
containing the text of the file.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Use \texttt{nltk{\linebreak[0]}.{\linebreak[0]}tokenize{\linebreak[0]}.{\linebreak[0]}regexp{\_}tokenize{\linebreak[0]}({\linebreak[0]})} to create a tokenizer that tokenizes
the various kinds of punctuation in this text.  Use a single
regular expression, with inline comments using the
\texttt{re{\linebreak[0]}.{\linebreak[0]}VERBOSE} flag.

\item {} 
Use \texttt{nltk{\linebreak[0]}.{\linebreak[0]}tokenize{\linebreak[0]}.{\linebreak[0]}regexp{\_}tokenize{\linebreak[0]}({\linebreak[0]})} to create a tokenizer that tokenizes
the following kinds of expression: monetary amounts; dates; names
of people and companies.

\end{list}

\item {} 
☼ Rewrite the following loop as a list comprehension:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = {[}\pysrcstring{'The'}, \pysrcstring{'dog'}, \pysrcstring{'gave'}, \pysrcstring{'John'}, \pysrcstring{'the'}, \pysrcstring{'newspaper'}{]}
\pysrcprompt{>{}>{}> }result = {[}{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} sent:
\pysrcmore{... }    word{\_}len = (word, len(word))
\pysrcmore{... }    result.append(word{\_}len)
\pysrcprompt{>{}>{}> }result
\pysrcoutput{{[}('The', 3), ('dog', 3), ('gave', 4), ('John', 4), ('the', 3), ('newspaper', 9){]}}}\end{alltt}

\item {} 
◑ Use the Porter Stemmer to normalize some tokenized text, calling
the stemmer on each word.  Do the same thing with the Lancaster Stemmer
and see if you observe any differences.

\item {} 
◑ Consider the numeric expressions in the following sentence from
the MedLine corpus: \docutilsroleexample{The corresponding free cortisol fractions in these
sera were 4.53 +/- 0.15{\%} and 8.16 +/- 0.23{\%}, respectively.}
Should we say that the numeric expression \docutilsroleexample{4.53 +/- 0.15{\%}} is three
words?  Or should we say that it's a single compound word?  Or should
we say that it is actually \emph{nine} words, since it's read ``four point
five three, plus or minus fifteen percent''?  Or should we say that
it's not a ``real'' word at all, since it wouldn't appear in any dictionary?
Discuss these different possibilities.  Can you think of application domains
that motivate at least two of these answers?

\item {} 
◑ Readability measures are used to score the reading difficulty of a
text, for the purposes of selecting texts of appropriate difficulty
for language learners.  Let us define
μ$_{w}$ to be the average number of letters per word, and
μ$_{s}$ to be the average number of words per sentence, in
a given text.  The Automated Readability Index (ARI) of the text
is defined to be:
\texttt{4{\linebreak[0]}.{\linebreak[0]}71{\linebreak[0]} {\linebreak[0]}*{\linebreak[0]} {\linebreak[0]}`{\linebreak[0]}`{\linebreak[0]} {\linebreak[0]}|{\linebreak[0]}mu{\linebreak[0]}|{\linebreak[0]}{\textbackslash}{\linebreak[0]} {\linebreak[0]}:{\linebreak[0]}subscript{\linebreak[0]}:{\linebreak[0]}`{\linebreak[0]}w{\linebreak[0]}`{\linebreak[0]} {\linebreak[0]}`{\linebreak[0]}`{\linebreak[0]}+{\linebreak[0]} {\linebreak[0]}0{\linebreak[0]}.{\linebreak[0]}5{\linebreak[0]} {\linebreak[0]}*{\linebreak[0]} {\linebreak[0]}`{\linebreak[0]}`{\linebreak[0]} {\linebreak[0]}|{\linebreak[0]}mu{\linebreak[0]}|{\linebreak[0]}{\textbackslash}{\linebreak[0]} {\linebreak[0]}:{\linebreak[0]}subscript{\linebreak[0]}:{\linebreak[0]}`{\linebreak[0]}s{\linebreak[0]}`{\linebreak[0]} {\linebreak[0]}`{\linebreak[0]}`{\linebreak[0]}-{\linebreak[0]} {\linebreak[0]}21{\linebreak[0]}.{\linebreak[0]}43}.
Compute the ARI score for various sections of the Brown Corpus, including
section \texttt{f} (popular lore) and \texttt{j} (learned).  Make use of the fact that
\texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}brown{\linebreak[0]}.{\linebreak[0]}words{\linebreak[0]}({\linebreak[0]})} produces a sequence of words, while
\texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}brown{\linebreak[0]}.{\linebreak[0]}sents{\linebreak[0]}({\linebreak[0]})} produces a sequence of sentences.

\item {} 
★ Obtain raw texts from two or more genres and compute their respective
reading difficulty scores as in the previous exercise.
E.g. compare ABC Rural News and ABC Science News (\texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}abc}).
Use Punkt to perform sentence segmentation.

\item {} 
★ Rewrite the following nested loop as a nested list comprehension:
\begin{quote}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }words = {[}\pysrcstring{'attribution'}, \pysrcstring{'confabulation'}, \pysrcstring{'elocution'},
\pysrcmore{... }         \pysrcstring{'sequoia'}, \pysrcstring{'tenacious'}, \pysrcstring{'unidirectional'}{]}
\pysrcprompt{>{}>{}> }vsequences = set()
\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} words:
\pysrcmore{... }    vowels = {[}{]}
\pysrcmore{... }    \pysrckeyword{for} char \pysrckeyword{in} word:
\pysrcmore{... }        \pysrckeyword{if} char \pysrckeyword{in} \pysrcstring{'aeiou'}:
\pysrcmore{... }            vowels.append(char)
\pysrcmore{... }    vsequences.add(\pysrcstring{'{}'}.join(vowels))
\pysrcprompt{>{}>{}> }sorted(vsequences)
\pysrcoutput{{[}'aiuio', 'eaiou', 'eouio', 'euoia', 'oauaio', 'uiieioa'{]}}}\end{alltt}
% sorted(set(''.join(c for c in word if c in 'aeiou') for word in words)) 
\end{quote}

\end{list}


%___________________________________________________________________________

\section{Counting Words: Several Interesting Applications}
\label{counting-words-several-interesting-applications}

Now that we can count words (tokens or types), we can write
programs to perform a variety of useful tasks, to study
stylistic differences in language use, differences between
languages, and even to generate random text.

Before getting started, we need to see how to get Python to
count the number of occurrences of \emph{each} word in a document.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }counts = nltk.defaultdict(int)           ①
\pysrcprompt{>{}>{}> }sec{\_}a = nltk.corpus.brown.words(categories=\pysrcstring{'a'})
\pysrcprompt{>{}>{}> }\pysrckeyword{for} token \pysrckeyword{in} sec{\_}a:
\pysrcmore{... }    counts{[}token{]} += 1                   ②
\pysrcprompt{>{}>{}> }\pysrckeyword{for} token \pysrckeyword{in} sorted(counts){[}:5{]}:         ③
\pysrcmore{... }    \pysrckeyword{print} counts{[}token{]}, token
\pysrcoutput{38 !}
\pysrcoutput{5 {\$}1}
\pysrcoutput{2 {\$}1,000}
\pysrcoutput{1 {\$}1,000,000,000}
\pysrcoutput{3 {\$}1,500}}\end{alltt}

In line \href{\#count-initialization}{①} we initialize the dictionary.
Then for each word in each sentence we increment a counter (line \href{\#count-increment}{②}).
To view the contents of the dictionary, we can iterate over its keys and print
each entry (here just for the first 5 entries, line \href{\#count-output}{③}).


%___________________________________________________________________________

\subsection{Frequency Distributions}
\label{frequency-distributions}

This style of output and our \texttt{counts} object
are just different forms of the same abstract structure
— a collection of items and their frequencies —
known as a \index{frequency distribution}\textbf{frequency distribution}.
Since we will often need to count things, NLTK provides
a \texttt{FreqDist{\linebreak[0]}({\linebreak[0]})} class.  We can write the same code more
conveniently as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fd = nltk.FreqDist(sec{\_}a)
\pysrcprompt{>{}>{}> }\pysrckeyword{for} token \pysrckeyword{in} sorted(fd){[}:5{]}:
\pysrcmore{... }    \pysrckeyword{print} fd{[}token{]}, token
\pysrcoutput{38 !}
\pysrcoutput{5 {\$}1}
\pysrcoutput{2 {\$}1,000}
\pysrcoutput{1 {\$}1,000,000,000}
\pysrcoutput{3 {\$}1,500}}\end{alltt}

Some of the methods defined on NLTK frequency distributions are shown in \href{\#freqdist}{Table 2.2}.
\hypertarget{freqdist}{}
\leavevmode
\begin{longtable}[c]{|p{0.123\locallinewidth}|p{0.219\locallinewidth}|p{0.604\locallinewidth}|}
\hline
\textbf{
Name
} & \textbf{
Sample
} & \textbf{
Description
} \\
\hline
\endhead

Count
 & 
\texttt{fd{\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}the{\linebreak[0]}'}{]}}
 & 
number of times a given sample occurred
 \\
\hline

Frequency
 & 
\texttt{fd{\linebreak[0]}.{\linebreak[0]}freq{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}the{\linebreak[0]}'})}
 & 
frequency of a given sample
 \\
\hline

N
 & 
\texttt{fd{\linebreak[0]}.{\linebreak[0]}N{\linebreak[0]}({\linebreak[0]})}
 & 
number of samples
 \\
\hline

Samples
 & 
\texttt{list{\linebreak[0]}({\linebreak[0]}fd{\linebreak[0]})}
 & 
list of distinct samples recorded (also \texttt{fd{\linebreak[0]}.{\linebreak[0]}keys{\linebreak[0]}({\linebreak[0]})})
 \\
\hline

Max
 & 
\texttt{fd{\linebreak[0]}.{\linebreak[0]}max{\linebreak[0]}({\linebreak[0]})}
 & 
sample with the greatest number of outcomes
 \\
\hline
\caption{
Frequency Distribution Module
}\end{longtable}

{\noindent}
This output isn't very interesting.
Perhaps it would be more informative to list the most frequent word tokens first.
Now a \texttt{FreqDist} object is just a kind of dictionary, so we can easily get
its key-value pairs and sort them by decreasing values, as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} operator \pysrckeyword{import} itemgetter
\pysrcprompt{>{}>{}> }sorted{\_}word{\_}counts = sorted(fd.items(), key=itemgetter(1), reverse=True)  ①
\pysrcprompt{>{}>{}> }{[}token \pysrckeyword{for} (token, freq) \pysrckeyword{in} sorted{\_}word{\_}counts{[}:20{]}{]}
\pysrcoutput{{[}'the', ',', '.', 'of', 'and', 'to', 'a', 'in', 'for', 'The', 'that',}
\pysrcoutput{'`{}`', 'is', 'was', "'{}'", 'on', 'at', 'with', 'be', 'by'{]}}}\end{alltt}

Note the arguments of the \texttt{sorted{\linebreak[0]}({\linebreak[0]})} function (line \href{\#sorted-args}{①}): \texttt{itemgetter{\linebreak[0]}({\linebreak[0]}1{\linebreak[0]})}
returns a function that can be called on any sequence object to return the item at position \texttt{1};
\texttt{reverse{\linebreak[0]}={\linebreak[0]}True} performs the sort in reverse order.  Together, these ensure that the
word with the highest frequency is listed first.  This reversed sort by frequency is
such a common requirement that it is built into the \texttt{FreqDist} object.
\href{\#print-freq}{Listing 2.2} demonstrates this, and also prints rank and cumulative frequency.
\hypertarget{print-freq}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{print{\_}freq}(tokens, num=50):
    fd = nltk.FreqDist(tokens)
    cumulative = 0.0
    rank = 0
    \pysrckeyword{for} word \pysrckeyword{in} fd.sorted(){[}:num{]}:
        rank += 1
        cumulative += fd{[}word{]} * 100.0 / fd.N()
        \pysrckeyword{print} \pysrcstring{"{\%}3d {\%}3.2d{\%}{\%} {\%}s"} {\%} (rank, cumulative, word)}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }print{\_}freq(nltk.corpus.brown.words(categories=\pysrcstring{'a'}), 20)
\pysrcoutput{  1  05{\%} the}
\pysrcoutput{  2  10{\%} ,}
\pysrcoutput{  3  14{\%} .}
\pysrcoutput{  4  17{\%} of}
\pysrcoutput{  5  19{\%} and}
\pysrcoutput{  6  21{\%} to}
\pysrcoutput{  7  23{\%} a}
\pysrcoutput{  8  25{\%} in}
\pysrcoutput{  9  26{\%} for}
\pysrcoutput{ 10  27{\%} The}
\pysrcoutput{ 11  28{\%} that}
\pysrcoutput{ 12  28{\%} `{}`}
\pysrcoutput{ 13  29{\%} is}
\pysrcoutput{ 14  30{\%} was}
\pysrcoutput{ 15  31{\%} '{}'}
\pysrcoutput{ 16  31{\%} on}
\pysrcoutput{ 17  32{\%} at}
\pysrcoutput{ 18  32{\%} with}
\pysrcoutput{ 19  33{\%} be}
\pysrcoutput{ 20  33{\%} by}}\end{alltt}
\caption{Words and Cumulative Frequencies, in Order of Decreasing Frequency}
\vspace{1ex}
\hrule
\end{pylisting}

Unfortunately the output in \href{\#print-freq}{Listing 2.2} is surprisingly dull.
A mere handful of tokens account for a third of the text.  They just represent
the plumbing of English text, and are completely uninformative!
How can we find words that are more indicative of a text?
As we will see in the exercises for this section, we can modify
the program to discard the non-content words.  In the next
section we see another approach.


%___________________________________________________________________________

\subsection{Stylistics}
\label{stylistics}

Stylistics is a broad term covering literary genres and varieties of language use.
Here we will look at a document collection that is categorized by genre,
and try to learn something about the patterns of word usage.
For example, \href{\#brown-modals}{Table 2.3} was constructed by counting
the number of times various modal words appear
in different sections of the corpus:
\hypertarget{brown-modals}{}
\leavevmode
\begin{longtable}[c]{|p{0.226\locallinewidth}|p{0.051\locallinewidth}|p{0.075\locallinewidth}|p{0.051\locallinewidth}|p{0.075\locallinewidth}|p{0.063\locallinewidth}|p{0.063\locallinewidth}|}
\hline
\textbf{
Genre
} & \textbf{
can
} & \textbf{
could
} & \textbf{
may
} & \textbf{
might
} & \textbf{
must
} & \textbf{
will
} \\
\hline
\endhead

skill and hobbies
 & 
273
 & 
59
 & 
130
 & 
22
 & 
83
 & 
259
 \\
\hline

humor
 & 
17
 & 
33
 & 
8
 & 
8
 & 
9
 & 
13
 \\
\hline

fiction: science
 & 
16
 & 
49
 & 
4
 & 
12
 & 
8
 & 
16
 \\
\hline

press: reportage
 & 
94
 & 
86
 & 
66
 & 
36
 & 
50
 & 
387
 \\
\hline

fiction: romance
 & 
79
 & 
195
 & 
11
 & 
51
 & 
46
 & 
43
 \\
\hline

religion
 & 
84
 & 
59
 & 
79
 & 
12
 & 
54
 & 
64
 \\
\hline
\caption{
Use of Modals in Brown Corpus, by Genre
}\end{longtable}

{\noindent}
Observe that the most frequent modal in the reportage genre is
\docutilsroleexample{will}, suggesting a focus on the future, while the most frequent
modal in the romance genre is \docutilsroleexample{could}, suggesting a focus on
possibilities.

We can also measure the lexical diversity of a genre, by
calculating the ratio of word types and word tokens, as
shown in \href{\#brown-types}{Table 2.4}.  Genres with lower diversity
have a higher number of tokens per type, thus we see that
humorous prose is almost twice as lexically diverse as
romance prose.
\hypertarget{brown-types}{}
\leavevmode
\begin{longtable}[c]{|p{0.226\locallinewidth}|p{0.145\locallinewidth}|p{0.133\locallinewidth}|p{0.075\locallinewidth}|}
\hline
\textbf{
Genre
} & \textbf{
Token Count
} & \textbf{
Type Count
} & \textbf{
Ratio
} \\
\hline
\endhead

skill and hobbies
 & 
82345
 & 
11935
 & 
6.9
 \\
\hline

humor
 & 
21695
 & 
5017
 & 
4.3
 \\
\hline

fiction: science
 & 
14470
 & 
3233
 & 
4.5
 \\
\hline

press: reportage
 & 
100554
 & 
14394
 & 
7.0
 \\
\hline

fiction: romance
 & 
70022
 & 
8452
 & 
8.3
 \\
\hline

religion
 & 
39399
 & 
6373
 & 
6.2
 \\
\hline
\caption{
Lexical Diversity of Various Genres in the Brown Corpus
}\end{longtable}

We can carry out a variety of interesting explorations simply by
counting words.  In fact, the field of \index{Corpus Linguistics}\textbf{Corpus Linguistics} focuses
heavily on creating and interpreting such tables of word counts.
\hypertarget{sec-defining-functions}{}

%___________________________________________________________________________

\subsection{Aside: Defining Functions}
\label{aside-defining-functions}
\label{sec-defining-functions}

It often happens that part of a program needs to be used several times
over.  For example, suppose we were writing a program that needed to
be able to form the plural of a singular noun, and that this needed to
be done at various places during the program.  Rather than repeating
the same code several times over, it is more efficient (and reliable)
to localize this work inside a \index{function}\textbf{function}.  A function is a
programming construct that can be called with one or more inputs and
which returns an output.  We define a function using the keyword
\texttt{def} followed by the function name and any input parameters,
followed by a colon; this in turn is followed by the body of the
function. We use the keyword \texttt{return} to indicate the value that is
produced as output by the function. The best way to convey this is
with an example. Our function \texttt{plural{\linebreak[0]}({\linebreak[0]})} in \href{\#plural}{Listing 2.3}
takes a singular noun and generates a plural form (one which is not always
correct).
\hypertarget{plural}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{plural}(word):
    \pysrckeyword{if} word.endswith(\pysrcstring{'y'}):
        return word{[}:-1{]} + \pysrcstring{'ies'}
    \pysrckeyword{elif} word{[}-1{]} \pysrckeyword{in} \pysrcstring{'sx'} \pysrckeyword{or} word{[}-2:{]} \pysrckeyword{in} {[}\pysrcstring{'sh'}, \pysrcstring{'ch'}{]}:
        return word + \pysrcstring{'es'}
    \pysrckeyword{elif} word.endswith(\pysrcstring{'an'}):
        return word{[}:-2{]} + \pysrcstring{'en'}
    return word + \pysrcstring{'s'}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }plural(\pysrcstring{'fairy'})
\pysrcoutput{'fairies'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }plural(\pysrcstring{'woman'})
\pysrcoutput{'women'}}\end{alltt}
\caption{Example of a Python function}
\vspace{1ex}
\hrule
\end{pylisting}

(There is much more to be said about ways of defining functions, but
we will defer this until \href{\#sec-functions}{Section 5.4}.)


%___________________________________________________________________________

\subsection{Lexical Dispersion}
\label{lexical-dispersion}

Word tokens vary in their distribution throughout a text.  We can
visualize word distributions to get an overall sense of topics and topic
shifts.  For example, consider the pattern of mention of the main
characters in Jane Austen's \emph{Sense and Sensibility}: Elinor, Marianne,
Edward and Willoughby.  The following plot contains four rows, one for
each name, in the order just given.  Each row contains a series of lines, drawn
to indicate the position of each token.
\begin{figure}[htbp]
\centering

\scalebox{0.514286}{\includegraphics{../images/words-dispersion.png}}
\caption{Lexical Dispersion Plot for the Main Characters in \emph{Sense and Sensibility}}\end{figure}

As you can see, \docutilsroleexample{Elinor} and \docutilsroleexample{Marianne} appear rather uniformly throughout
the text, while \docutilsroleexample{Edward} and \docutilsroleexample{Willoughby} tend to appear separately.
Here is the code that generated the above plot.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }names = {[}\pysrcstring{'Elinor'}, \pysrcstring{'Marianne'}, \pysrcstring{'Edward'}, \pysrcstring{'Willoughby'}{]}
\pysrcprompt{>{}>{}> }text = nltk.corpus.gutenberg.words(\pysrcstring{'austen-sense.txt'})
\pysrcprompt{>{}>{}> }nltk.draw.dispersion{\_}plot(text, names)}\end{alltt}


%___________________________________________________________________________

\subsection{Comparing Word Lengths in Different Languages}
\label{comparing-word-lengths-in-different-languages}

We can use a frequency distribution to examine the distribution
of word lengths in a corpus.  For each word, we find its length,
and increment the count for words of this length.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{print{\_}length{\_}dist}(text):
\pysrcmore{... }    fd = nltk.FreqDist(len(token) \pysrckeyword{for} token \pysrckeyword{in} text \pysrckeyword{if} re.match(r\pysrcstring{'{\textbackslash}w+{\$}'}, token))
\pysrcmore{... }    \pysrckeyword{for} i \pysrckeyword{in} range(1,15):
\pysrcmore{... }        \pysrckeyword{print} \pysrcstring{"{\%}2d"} {\%} int(100*fd.freq(i)),
\pysrcmore{... }    \pysrckeyword{print}}\end{alltt}

{\noindent}
Now we can call \texttt{print{\_}length{\_}dist} on a text to print the distribution of
word lengths.  We see that the most frequent word length for the English
sample is 3 characters, while the most frequent length for the Finnish
sample is 5-6 characters.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }print{\_}length{\_}dist(nltk.corpus.genesis.words(\pysrcstring{'english-kjv.txt'}))
\pysrcoutput{ 2 15 30 23 12  6  4  2  1  0  0  0  0  0}
\pysrcoutput{}\pysrcprompt{>{}>{}> }print{\_}length{\_}dist(nltk.corpus.genesis.words(\pysrcstring{'finnish.txt'}))
\pysrcoutput{ 0 12  6 10 17 17 11  9  5  3  2  1  0  0}}\end{alltt}

This is an intriguing area for exploration, and so in \href{\#word-len-dist}{Listing 2.4}
we look at it on a larger scale using the Universal Declaration of Human Rights corpus,
which has text samples from over 300 languages.
(Note that the names of the files in this corpus include
information about character encoding; here we will use texts in ISO Latin-1.)
The output is shown in \href{\#fig-word-len-dist}{Figure 2.3} (a color figure in the online version).
\hypertarget{word-len-dist}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{import} pylab

\pysrckeyword{def} \pysrcdefname{cld}(lang):
    text = nltk.corpus.udhr.words(lang)
    fd = nltk.FreqDist(len(token) \pysrckeyword{for} token \pysrckeyword{in} text)
    ld = {[}100*fd.freq(i) \pysrckeyword{for} i \pysrckeyword{in} range(36){]}
    return {[}sum(ld{[}0:i+1{]}) \pysrckeyword{for} i \pysrckeyword{in} range(len(ld)){]}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }langs = {[}\pysrcstring{'Chickasaw-Latin1'}, \pysrcstring{'English-Latin1'},
\pysrcmore{... }         \pysrcstring{'German{\_}Deutsch-Latin1'}, \pysrcstring{'Greenlandic{\_}Inuktikut-Latin1'},
\pysrcmore{... }         \pysrcstring{'Hungarian{\_}Magyar-Latin1'}, \pysrcstring{'Ibibio{\_}Efik-Latin1'}{]}
\pysrcprompt{>{}>{}> }dists = {[}pylab.plot(cld(l), label=l{[}:-7{]}, linewidth=2) \pysrckeyword{for} l \pysrckeyword{in} langs{]}
\pysrcprompt{>{}>{}> }pylab.title(\pysrcstring{'Cumulative Word Length Distributions for Several Languages'})
\pysrcprompt{>{}>{}> }pylab.legend(loc=\pysrcstring{'lower right'})
\pysrcprompt{>{}>{}> }pylab.show()     }\end{alltt}
\caption{Cumulative Word Length Distributions for Several Languages}
\vspace{1ex}
\hrule
\end{pylisting}
\hypertarget{fig-word-len-dist}{}\begin{figure}[htbp]
\centering

\scalebox{0.128571}{\includegraphics{../images/word-len-dist.png}}
\caption{Cumulative Word Length Distributions for Several Languages}\end{figure}


%___________________________________________________________________________

\subsection{Generating Random Text with Style}
\label{generating-random-text-with-style}

We have used frequency distributions to count the number of occurrences of
each word in a text.  Here we will generalize this idea to look at the
distribution of words in a given context.
A \index{conditional frequency distribution}\textbf{conditional frequency distribution} is a collection
of frequency distributions, each one for a different condition.
Here the condition will be the preceding word.

In \href{\#random}{Listing 2.5}, we've defined a function \texttt{train{\_}model{\linebreak[0]}({\linebreak[0]})} that
uses \texttt{ConditionalFreqDist{\linebreak[0]}({\linebreak[0]})} to count words as they appear
relative to the context defined by the preceding word (stored in \texttt{prev}).
It scans the corpus, incrementing the appropriate counter, and
updating the value of \texttt{prev}.  The function \texttt{generate{\_}model{\linebreak[0]}({\linebreak[0]})}
contains a simple loop to generate text: we set an initial
context, pick the most likely token in that context as our next
word (using \texttt{max{\linebreak[0]}({\linebreak[0]})}), and then use that word as our new context.
This simple approach to text generation tends to get stuck in loops;
another method would be to randomly choose the next word from among
the available words.
\hypertarget{random}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{train{\_}model}(text):
    cfdist = nltk.ConditionalFreqDist()
    prev = None
    \pysrckeyword{for} word \pysrckeyword{in} text:
        cfdist{[}prev{]}.inc(word)
        prev = word
    return cfdist

\pysrckeyword{def} \pysrcdefname{generate{\_}model}(cfdist, word, num=15):
    \pysrckeyword{for} i \pysrckeyword{in} range(num):
        \pysrckeyword{print} word,
        word = cfdist{[}word{]}.max()}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }model = train{\_}model(nltk.corpus.genesis.words(\pysrcstring{'english-kjv.txt'}))
\pysrcprompt{>{}>{}> }model{[}\pysrcstring{'living'}{]}
\pysrcoutput{<FreqDist with 16 samples>}
\pysrcoutput{}\pysrcprompt{>{}>{}> }list(model{[}\pysrcstring{'living'}{]})
\pysrcoutput{{[}'substance', ',', '.', 'thing', 'soul', 'creature'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }generate{\_}model(model, \pysrcstring{'living'})
\pysrcoutput{living creature that he said , and the land of the land of the land}}\end{alltt}
\caption{Generating Random Text in the Style of Genesis}
\vspace{1ex}
\hrule
\end{pylisting}
% TODO: simple concordancing 


%___________________________________________________________________________

\subsection{Collocations}
\label{collocations}

Collocations are pairs of content words that occur together more
often than one would expect if the words of a document were scattered randomly.
We can find collocations by counting how many times a pair of words
\emph{w}$_{1}$, \emph{w}$_{2}$ occurs together, compared
to the overall counts of these words (this program uses
a heuristic related to the \index{mutual information}\textbf{mutual information} measure,
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}www{\linebreak[0]}.{\linebreak[0]}collocations{\linebreak[0]}.{\linebreak[0]}de{\linebreak[0]}/})
In \href{\#id17}{Listing 2.6} we try this for the files in the webtext corpus.
\hypertarget{id17}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{collocations}(words):
    \pysrckeyword{from} operator \pysrckeyword{import} itemgetter

    \pysrccomment{{\#} Count the words and bigrams}
    wfd = nltk.FreqDist(words)
    pfd = nltk.FreqDist(tuple(words{[}i:i+2{]}) \pysrckeyword{for} i \pysrckeyword{in} range(len(words)-1))

    \pysrccomment{{\#}}
    scored = {[}((w1,w2), score(w1, w2, wfd, pfd)) \pysrckeyword{for} w1, w2 \pysrckeyword{in} pfd{]}
    scored.sort(key=itemgetter(1), reverse=True)
    return map(itemgetter(0), scored)

\pysrckeyword{def} \pysrcdefname{score}(word1, word2, wfd, pfd, power=3):
    freq1 = wfd{[}word1{]}
    freq2 = wfd{[}word2{]}
    freq12 = pfd{[}(word1, word2){]}
    return freq12 ** power / float(freq1 * freq2)}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} file \pysrckeyword{in} nltk.corpus.webtext.files():
\pysrcmore{... }    words = {[}word.lower() \pysrckeyword{for} word \pysrckeyword{in} nltk.corpus.webtext.words(file) \pysrckeyword{if} len(word) > 2{]}
\pysrcmore{... }    \pysrckeyword{print} file, {[}w1+\pysrcstring{' '}+w2 \pysrckeyword{for} w1, w2 \pysrckeyword{in} collocations(words){[}:15{]}{]}
\pysrcoutput{overheard {[}'new york', 'teen boy', 'teen girl', 'you know', 'middle aged',}
\pysrcoutput{'flight attendant', 'puerto rican', 'last night', 'little boy', 'taco bell',}
\pysrcoutput{'statue liberty', 'bus driver', 'ice cream', 'don know', 'high school'{]}}
\pysrcoutput{pirates {[}'jack sparrow', 'will turner', 'elizabeth swann', 'davy jones',}
\pysrcoutput{'flying dutchman', 'lord cutler', 'cutler beckett', 'black pearl', 'tia dalma',}
\pysrcoutput{'heh heh', 'edinburgh trader', 'port royal', 'bamboo pole', 'east india', 'jar dirt'{]}}
\pysrcoutput{singles {[}'non smoker', 'would like', 'dining out', 'like meet', 'age open',}
\pysrcoutput{'sense humour', 'looking for', 'social drinker', 'down earth', 'long term',}
\pysrcoutput{'quiet nights', 'easy going', 'medium build', 'nights home', 'weekends away'{]}}
\pysrcoutput{wine {[}'high toned', 'top ***', 'not rated', 'few years', 'medium weight',}
\pysrcoutput{'year two', 'cigar box', 'cote rotie', 'mixed feelings', 'demi sec',}
\pysrcoutput{'from half', 'brown sugar', 'bare ****', 'tightly wound', 'sous bois'{]}}}\end{alltt}
\caption{A Simple Program to Find Collocations}
\vspace{1ex}
\hrule
\end{pylisting}
% TODO: add section on Distributional Similarity 


%___________________________________________________________________________

\subsection{Exercises}
\label{id18}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☺ Compare the lexical dispersion plot with Google Trends, which
shows the frequency with which a term has been referenced in news reports
or been used in search terms over time.

\item {} 
☼ Pick a text, and explore the dispersion of particular words.  What
does this tell you about the words, or the text?

\item {} 
☼ The program in \href{\#print-freq}{Listing 2.2} used a dictionary of word counts.
Modify the code that creates these word counts so that it ignores non-content
words.  You can easily get a list of words to ignore with:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }ignored{\_}words = nltk.corpus.stopwords.words(\pysrcstring{'english'})}\end{alltt}

\item {} 
☼ Modify the \texttt{generate{\_}model{\linebreak[0]}({\linebreak[0]})} function in \href{\#random}{Listing 2.5} to use Python's
\texttt{random{\linebreak[0]}.{\linebreak[0]}choose{\linebreak[0]}({\linebreak[0]})} method to randomly pick the next word from
the available set of words.

\item {} 
☼ \textbf{The demise of teen language:}
Read the BBC News article: \emph{UK's Vicky Pollards 'left behind'} \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}news{\linebreak[0]}.{\linebreak[0]}bbc{\linebreak[0]}.{\linebreak[0]}co{\linebreak[0]}.{\linebreak[0]}uk{\linebreak[0]}/{\linebreak[0]}1{\linebreak[0]}/{\linebreak[0]}hi{\linebreak[0]}/{\linebreak[0]}education{\linebreak[0]}/{\linebreak[0]}6173441{\linebreak[0]}.{\linebreak[0]}stm}.
The article gives the following statistic about teen language:
``the top 20 words used, including yeah, no, but and like, account for around a third of all words.''
Use the program in \href{\#print-freq}{Listing 2.2} to find out how many word types account for a third
of all word tokens, for a variety of text sources.  What do you conclude about this statistic?
Read more about this on \emph{LanguageLog}, at \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}itre{\linebreak[0]}.{\linebreak[0]}cis{\linebreak[0]}.{\linebreak[0]}upenn{\linebreak[0]}.{\linebreak[0]}edu{\linebreak[0]}/{\linebreak[0]}{\textasciitilde}{\linebreak[0]}myl{\linebreak[0]}/{\linebreak[0]}languagelog{\linebreak[0]}/{\linebreak[0]}archives{\linebreak[0]}/{\linebreak[0]}003993{\linebreak[0]}.{\linebreak[0]}html}.

\item {} 
◑ Write a program to find all words that occur at least three times in the Brown Corpus.

\item {} 
◑ Write a program to generate a table of token/type ratios, as we saw in
\href{\#brown-types}{Table 2.4}.  Include the full set of Brown Corpus genres (\texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}brown{\linebreak[0]}.{\linebreak[0]}categories{\linebreak[0]}({\linebreak[0]})}).
Which genre has the lowest diversity (greatest number of tokens per type)?
Is this what you would have expected?

\item {} 
◑ Modify the text generation program in \href{\#random}{Listing 2.5} further, to
do the following tasks:
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Store the \emph{n} most likely words in a list \texttt{lwords} then randomly
choose a word from the list using \texttt{random{\linebreak[0]}.{\linebreak[0]}choice{\linebreak[0]}({\linebreak[0]})}.

\item {} 
Select a particular genre, such as a section of the Brown Corpus,
or a genesis translation, one of the Gutenberg texts, or one of the Web texts.  Train
the model on this corpus and get it to generate random text.  You
may have to experiment with different start words. How intelligible
is the text?  Discuss the strengths and weaknesses of this method of
generating random text.

\item {} 
Now train your system using two distinct genres and experiment
with generating text in the hybrid genre.  Discuss your observations.

\end{list}

\item {} 
◑ Write a program to print the most frequent bigrams
(pairs of adjacent words) of a text,
omitting non-content words, in order of decreasing frequency.

\item {} 
◑ Write a program to create a table of word frequencies by genre,
like the one given above for modals.  Choose your own words and
try to find words whose presence (or absence) is typical of a genre.
Discuss your findings.

\item {} 
◑ \textbf{Zipf's Law}:
Let \emph{f(w)} be the frequency of a word \emph{w} in free text. Suppose that
all the words of a text are ranked according to their frequency,
with the most frequent word first. Zipf's law states that the
frequency of a word type is inversely proportional to its rank
(i.e. \emph{f.r = k}, for some constant \emph{k}). For example, the 50th most
common word type should occur three times as frequently as the
150th most common word type.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Write a function to process a large text and plot word
frequency against word rank using \texttt{pylab{\linebreak[0]}.{\linebreak[0]}plot}. Do
you confirm Zipf's law? (Hint: it helps to use a logarithmic scale).
What is going on at the extreme ends of the plotted line?

\item {} 
Generate random text, e.g. using \texttt{random{\linebreak[0]}.{\linebreak[0]}choice{\linebreak[0]}(\pysrcstring{"{\linebreak[0]}abcdefg{\linebreak[0]} {\linebreak[0]}"})},
taking care to include the space character.  You will need to
\texttt{\pysrckeyword{import} {\linebreak[0]}random} first.  Use the string
concatenation operator to accumulate characters into a (very)
long string.  Then tokenize this string, and generate the Zipf
plot as before, and compare the two plots.  What do you make of
Zipf's Law in the light of this?

\end{list}

\item {} 
◑ \textbf{Exploring text genres:}
Investigate the table of modal distributions and look for other patterns.
Try to explain them in terms of your own impressionistic understanding
of the different genres.  Can you find other closed classes of words that
exhibit significant differences across different genres?

\item {} 
◑ Write a function \texttt{tf{\linebreak[0]}({\linebreak[0]})} that takes a word and the name of a section
of the Brown Corpus as arguments, and computes the text frequency of the word
in that section of the corpus.

\item {} 
★ \textbf{Authorship identification:}
Reproduce some of the results of \cite{zhao07}.

\item {} 
★ \textbf{Gender-specific lexical choice:}
Reproduce some of the results of \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}www{\linebreak[0]}.{\linebreak[0]}clintoneast{\linebreak[0]}.{\linebreak[0]}com{\linebreak[0]}/{\linebreak[0]}articles{\linebreak[0]}/{\linebreak[0]}words{\linebreak[0]}.{\linebreak[0]}php}

\end{list}


%___________________________________________________________________________

\section{WordNet: An English Lexical Database}
\label{wordnet-an-english-lexical-database}

\index{WordNet}\textit{WordNet} is a semantically-oriented dictionary of English,
similar to a traditional thesaurus but with a richer structure.
WordNet groups words into synonym sets, or \index{synsets}\textbf{synsets}, each with
its own definition and with links to other synsets.
WordNet 3.0 data is distributed with NLTK, and includes 117,659 synsets.

Although WordNet was originally developed for research
in psycholinguistics, it is widely used in NLP and Information Retrieval.
WordNets are being developed for many other languages, as documented
at \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}www{\linebreak[0]}.{\linebreak[0]}globalwordnet{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/}.


%___________________________________________________________________________

\subsection{Senses and Synonyms}
\label{senses-and-synonyms}

Consider the following sentence:
\hypertarget{carex1}{}\begin{itemize}
\item[(2)] 
Benz is credited with the invention of the motorcar.
\end{itemize}

If we replace \docutilsroleexample{motorcar} in \href{\#carex1}{(2)} by \docutilsroleexample{automobile}, the
meaning of the sentence stays pretty much the same:
\hypertarget{carex2}{}\begin{itemize}
\item[(3)] 
Benz is credited with the invention of the automobile.
\end{itemize}

Since everything else in the sentence has remained unchanged, we can
conclude that the words \docutilsroleexample{motorcar} and \docutilsroleexample{automobile} have the
same meaning, i.e. they are \index{synonyms}\textbf{synonyms}.

In order to look up the senses of a word, we need to pick
a part of speech for the word.  WordNet contains four dictionaries: \texttt{N}
(nouns), \texttt{V} (verbs), \texttt{ADJ} (adjectives), and \texttt{ADV}
(adverbs). To simplify our discussion, we will focus on the \texttt{N}
dictionary here.  Let's look up \docutilsroleexample{motorcar} in the \texttt{N} dictionary.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk \pysrckeyword{import} wordnet
\pysrcprompt{>{}>{}> }car = wordnet.N{[}\pysrcstring{'motorcar'}{]}
\pysrcprompt{>{}>{}> }car
\pysrcoutput{motorcar (noun)}}\end{alltt}

The variable \texttt{car} is now bound to a \texttt{Word} object.
Words will often have more than sense, where each
sense is represented by a synset. However,
\docutilsroleexample{motorcar} only has one sense in WordNet, as we can discover
using \texttt{len{\linebreak[0]}({\linebreak[0]})}.  We can then find the synset (a set of
lemmas), the words it contains, and a gloss.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }len(car)
\pysrcoutput{1}
\pysrcoutput{}\pysrcprompt{>{}>{}> }car{[}0{]}
\pysrcoutput{{\{}noun: car, auto, automobile, machine, motorcar{\}}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }list(car{[}0{]})
\pysrcoutput{{[}'car', 'auto', 'automobile', 'machine', 'motorcar'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }car{[}0{]}.gloss
\pysrcoutput{'a motor vehicle with four wheels; usually propelled by an}
\pysrcoutput{internal combustion engine;}
\pysrcoutput{"he needs a car to get to work"'}}\end{alltt}

The \texttt{wordnet} module also defines \texttt{Synset}s.
Let's look at a word which is \index{polysemous}\textbf{polysemous};
that is, which has multiple synsets:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }poly = wordnet.N{[}\pysrcstring{'pupil'}{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} synset \pysrckeyword{in} poly:
\pysrcmore{... }    \pysrckeyword{print} synset
\pysrcoutput{{\{}noun: student, pupil, educatee{\}}}
\pysrcoutput{{\{}noun: pupil{\}}}
\pysrcoutput{{\{}noun: schoolchild, school-age{\_}child, pupil{\}}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }poly{[}1{]}.gloss
\pysrcoutput{'the contractile aperture in the center of the iris of the eye;}
\pysrcoutput{resembles a large black dot'}}\end{alltt}


%___________________________________________________________________________

\subsection{The WordNet Hierarchy}
\label{the-wordnet-hierarchy}

WordNet synsets correspond to abstract concepts, which may or may not
have corresponding words in English.  These concepts are linked together in a hierarchy.
Some are very general, such as \emph{Entity}, \emph{State}, \emph{Event} — these are called
\index{unique beginners}\textbf{unique beginners}.  Others, such as \emph{gas guzzler} and
\emph{hatchback}, are much more specific. A small portion of a concept
hierarchy is illustrated in \href{\#wn-hierarchy}{Figure 2.4}. The edges between nodes
indicate the hypernym/hyponym relation; the dotted line at the top is
intended to indicate that \emph{artifact} is a non-immediate hypernym of \emph{motorcar}.
\hypertarget{wn-hierarchy}{}\begin{figure}[htbp]
\centering

\scalebox{0.077143}{\includegraphics{../images/wordnet-hierarchy.png}}
\caption{Fragment of WordNet Concept Hierarchy}\end{figure}

WordNet makes it easy to navigate between concepts.
For example, given a concept like \emph{motorcar},
we can look at the concepts that are more specific;
the (immediate) \index{hyponyms}\textbf{hyponyms}. Here is one way to carry out this
navigation:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} concept \pysrckeyword{in} car{[}0{]}{[}wordnet.HYPONYM{]}{[}:10{]}:
\pysrcmore{... }        \pysrckeyword{print} concept
\pysrcoutput{{\{}noun: ambulance{\}}}
\pysrcoutput{{\{}noun: beach{\_}wagon, station{\_}wagon, wagon, estate{\_}car, beach{\_}waggon, station{\_}waggon, waggon{\}}}
\pysrcoutput{{\{}noun: bus, jalopy, heap{\}}}
\pysrcoutput{{\{}noun: cab, hack, taxi, taxicab{\}}}
\pysrcoutput{{\{}noun: compact, compact{\_}car{\}}}
\pysrcoutput{{\{}noun: convertible{\}}}
\pysrcoutput{{\{}noun: coupe{\}}}
\pysrcoutput{{\{}noun: cruiser, police{\_}cruiser, patrol{\_}car, police{\_}car, prowl{\_}car, squad{\_}car{\}}}
\pysrcoutput{{\{}noun: electric, electric{\_}automobile, electric{\_}car{\}}}
\pysrcoutput{{\{}noun: gas{\_}guzzler{\}}}}\end{alltt}

{\noindent}
We can also move up the hierarchy, by looking at broader concepts than
\emph{motorcar}, e.g. the immediate \index{hypernym}\textbf{hypernym} of a concept:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }car{[}0{]}{[}wordnet.HYPERNYM{]}
\pysrcoutput{{[}{\{}noun: motor{\_}vehicle, automotive{\_}vehicle{\}}{]}}}\end{alltt}

We can also look for the hypernyms of hypernyms.  In fact, from any
synset we can trace (multiple) paths back to a unique beginner.
Synsets have a method for doing this, called \texttt{tree{\linebreak[0]}({\linebreak[0]})},
which produces a nested list structure.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }pprint.pprint(wordnet.N{[}\pysrcstring{'car'}{]}{[}0{]}.tree(wordnet.HYPERNYM))
\pysrcoutput{{[}{\{}noun: car, auto, automobile, machine, motorcar{\}},}
\pysrcoutput{ {[}{\{}noun: motor{\_}vehicle, automotive{\_}vehicle{\}},}
\pysrcoutput{  {[}{\{}noun: self-propelled{\_}vehicle{\}},}
\pysrcoutput{   {[}{\{}noun: wheeled{\_}vehicle{\}},}
\pysrcoutput{    {[}{\{}noun: vehicle{\}},}
\pysrcoutput{     {[}{\{}noun: conveyance, transport{\}},}
\pysrcoutput{      {[}{\{}noun: instrumentality, instrumentation{\}},}
\pysrcoutput{       {[}{\{}noun: artifact, artefact{\}},}
\pysrcoutput{        {[}{\{}noun: whole, unit{\}},}
\pysrcoutput{         {[}{\{}noun: object, physical{\_}object{\}},}
\pysrcoutput{          {[}{\{}noun: physical{\_}entity{\}}, {[}{\{}noun: entity{\}}{]}{]}{]}{]}{]}{]}{]}{]},}
\pysrcoutput{    {[}{\{}noun: container{\}},}
\pysrcoutput{     {[}{\{}noun: instrumentality, instrumentation{\}},}
\pysrcoutput{      {[}{\{}noun: artifact, artefact{\}},}
\pysrcoutput{       {[}{\{}noun: whole, unit{\}},}
\pysrcoutput{        {[}{\{}noun: object, physical{\_}object{\}},}
\pysrcoutput{         {[}{\{}noun: physical{\_}entity{\}}, {[}{\{}noun: entity{\}}{]}{]}{]}{]}{]}{]}{]}{]}{]}{]}{]}}}\end{alltt}

A related method \texttt{closure{\linebreak[0]}({\linebreak[0]})} produces a flat version of this structure,
with repeats eliminated.
Both of these functions take an optional \texttt{depth} argument that permits
us to limit the number of steps to take.
(This is important when using unbounded relations like \texttt{SIMILAR}.)
\href{\#wordnet-rel}{Table 2.5} lists the most important lexical relations supported
by WordNet; see \texttt{dir{\linebreak[0]}({\linebreak[0]}wordnet{\linebreak[0]})} for a full list.
\hypertarget{wordnet-rel}{}
\leavevmode
\begin{longtable}[c]{|p{0.145\locallinewidth}|p{0.203\locallinewidth}|p{0.551\locallinewidth}|}
\hline

Hypernym
 & 
more general
 & 
\docutilsroleexample{animal} is a hypernym of \docutilsroleexample{dog}
 \\
\hline

Hyponym
 & 
more specific
 & 
\docutilsroleexample{dog} is a hyponym of \docutilsroleexample{animal}
 \\
\hline

Meronym
 & 
part of
 & 
\docutilsroleexample{door} is a meronym of \docutilsroleexample{house}
 \\
\hline

Holonym
 & 
has part
 & 
\docutilsroleexample{house} is a holonym of \docutilsroleexample{door}
 \\
\hline

Synonym
 & 
similar meaning
 & 
\docutilsroleexample{car} is a synonym of \docutilsroleexample{automobile}
 \\
\hline

Antonym
 & 
opposite meaning
 & 
\titlereference{like} is an antonym of \docutilsroleexample{dislike}
 \\
\hline

Entailment
 & 
necessary action
 & 
\titlereference{step} is an entailment of \docutilsroleexample{walk}
 \\
\hline
\caption{
Major WordNet Lexical Relations
}\end{longtable}

Recall that we can iterate over the words of a synset, with \texttt{\pysrckeyword{for} {\linebreak[0]}word{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}synset}.
We can also test if a word is in a dictionary, e.g. \texttt{\pysrckeyword{if} {\linebreak[0]}word{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}wordnet{\linebreak[0]}.{\linebreak[0]}V}.
As our last task, let's put these together to find ``animal words'' that are used as verbs.
Since there are a lot of these, we will cut this off at depth 4.
Can you think of the animal and verb sense of each word?
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }animals = wordnet.N{[}\pysrcstring{'animal'}{]}{[}0{]}.closure(wordnet.HYPONYM, depth=4)
\pysrcprompt{>{}>{}> }{[}word \pysrckeyword{for} synset \pysrckeyword{in} animals \pysrckeyword{for} word \pysrckeyword{in} synset \pysrckeyword{if} word \pysrckeyword{in} wordnet.V{]}
\pysrcoutput{{[}'pet', 'stunt', 'prey', 'quarry', 'game', 'mate', 'head', 'dog',}
\pysrcoutput{ 'stray', 'dam', 'sire', 'steer', 'orphan', 'spat', 'sponge',}
\pysrcoutput{ 'worm', 'grub', 'pooch', 'toy', 'queen', 'baby', 'pup', 'whelp',}
\pysrcoutput{ 'cub', 'kit', 'kitten', 'foal', 'lamb', 'fawn', 'bird', 'grouse',}
\pysrcoutput{ 'hound', 'bulldog', 'stud', 'hog', 'baby', 'fish', 'cock', 'parrot',}
\pysrcoutput{ 'frog', 'beetle', 'bug', 'bug', 'queen', 'leech', 'snail', 'slug',}
\pysrcoutput{ 'clam', 'cockle', 'oyster', 'scallop', 'scollop', 'escallop', 'quail'{]}}}\end{alltt}

NLTK also includes VerbNet, a hierarhical verb lexicon linked to WordNet.
It can be accessed with \texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}verbnet}.


%___________________________________________________________________________

\subsection{WordNet Similarity}
\label{wordnet-similarity}
% TODO: discuss WSD, mention Semcor, give pine cone example 
% It is often useful to be able to tell whether two lexical concepts are
% `semantically related`:dt:. For example, in order to check whether a
% particular instance of the word `bank`:lx: means
% *financial institution*, we can count the number of nearby words
% that are semantically related to this sense.  Using WordNet, we can
% investigate whether semantic relatedness can be
% expressed in terms of the graph structure of the concept hierarchy. 

We would expect that the semantic similarity of two concepts
would correlate with the length of the path between them in WordNet.
The \texttt{wordnet} package includes a variety of measures
that incorporate this basic insight.  For example,
\texttt{path{\_}similarity} assigns a score in the range 0–1, based on the shortest path that connects the concepts in the hypernym
hierarchy (-1 is returned in those cases where a path cannot be
found).  A score of 1 represents identity, i.e., comparing a sense with
itself will return 1.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }wordnet.N{[}\pysrcstring{'poodle'}{]}{[}0{]}.path{\_}similarity(wordnet.N{[}\pysrcstring{'dalmatian'}{]}{[}1{]})
\pysrcoutput{0.33333333333333331}
\pysrcoutput{}\pysrcprompt{>{}>{}> }wordnet.N{[}\pysrcstring{'dog'}{]}{[}0{]}.path{\_}similarity(wordnet.N{[}\pysrcstring{'cat'}{]}{[}0{]})
\pysrcoutput{0.20000000000000001}
\pysrcoutput{}\pysrcprompt{>{}>{}> }wordnet.V{[}\pysrcstring{'run'}{]}{[}0{]}.path{\_}similarity(wordnet.V{[}\pysrcstring{'walk'}{]}{[}0{]})
\pysrcoutput{0.25}
\pysrcoutput{}\pysrcprompt{>{}>{}> }wordnet.V{[}\pysrcstring{'run'}{]}{[}0{]}.path{\_}similarity(wordnet.V{[}\pysrcstring{'think'}{]}{[}0{]})
\pysrcoutput{-1}}\end{alltt}

Several other similarity measures are provided in \texttt{wordnet}:
Leacock-Chodorow, Wu-Palmer, Resnik, Jiang-Conrath, and Lin.
For a detailed comparison of various measures, see \cite{budanitsky2006ewb}.


%___________________________________________________________________________

\subsection{Exercises}
\label{id21}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Familiarize yourself with the WordNet interface, by reading the
documentation available via \texttt{help{\linebreak[0]}({\linebreak[0]}wordnet{\linebreak[0]})}.  Try out the text-based
browser, \texttt{wordnet{\linebreak[0]}.{\linebreak[0]}browse{\linebreak[0]}({\linebreak[0]})}.

\item {} 
☼ Investigate the holonym / meronym relations for some nouns.  Note that there
are three kinds (member, part, substance), so access is more specific,
e.g., \texttt{wordnet{\linebreak[0]}.{\linebreak[0]}MEMBER{\_}MERONYM}, \texttt{wordnet{\linebreak[0]}.{\linebreak[0]}SUBSTANCE{\_}HOLONYM}.

\end{list}
% Currently too hard since we can't iterate over synsets easily
% #. |soso| What percentage of noun synsets have no hyponyms? 
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ The polysemy of a word is the number of senses it has.
Using WordNet, we can determine that the noun \emph{dog} has 7 senses
with: \texttt{len{\linebreak[0]}({\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}wordnet{\linebreak[0]}.{\linebreak[0]}N{\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}dog{\linebreak[0]}'}{]}{\linebreak[0]})}.
Compute the average polysemy of nouns, verbs, adjectives and
adverbs according to WordNet.

\item {} 
◑ What is the branching factor of the noun hypernym hierarchy?
(For all noun synsets that have hyponyms, how many do they have on average?)

\item {} 
◑ Define a function \texttt{supergloss{\linebreak[0]}({\linebreak[0]}s{\linebreak[0]})} that takes a synset \texttt{s} as its argument
and returns a string consisting of the concatenation of the glosses of \texttt{s}, all
hypernyms of \texttt{s}, and all hyponyms of \texttt{s}.

\item {} 
◑ Write a program to score the similarity of two nouns as the depth
of their first common hypernym.

\item {} 
★ Use one of the predefined similarity measures to score
the similarity of each of the following pairs of words.
Rank the pairs in order of decreasing similarity.
How close is your ranking to the order given here?
(Note that this order was established experimentally
by \cite{millercharles1998}.)

\end{list}
\begin{description}
\item[{::}] \leavevmode 
car-automobile, gem-jewel, journey-voyage, boy-lad,
coast-shore, asylum-madhouse, magician-wizard, midday-noon,
furnace-stove, food-fruit, bird-cock, bird-crane, tool-implement,
brother-monk, lad-brother, crane-implement, journey-car,
monk-oracle, cemetery-woodland, food-rooster, coast-hill,
forest-graveyard, shore-woodland, monk-slave, coast-forest,
lad-wizard, chord-smile, glass-magician, rooster-voyage, noon-string.

\end{description}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
★ Write a program that processes a text and discovers
cases where a word has been used with a novel sense.
For each word, compute the wordnet similarity
between all synsets of the word and all synsets of the
words in its context.  (Note that this is a crude
approach; doing it well is an open research problem.)

\end{list}


%___________________________________________________________________________

\section{Conclusion}
\label{conclusion}

In this chapter we saw that we can do a variety of interesting
language processing tasks that focus solely on words.  Tokenization
turns out to be far more difficult than expected.
No single solution works well across-the-board, and we
must decide what counts as a token depending on the application
domain.  We also looked at normalization (including lemmatization) and
saw how it collapses distinctions between tokens.  In the next chapter
we will look at word classes and automatic tagging.


%___________________________________________________________________________

\section{Summary}
\label{id23}
\begin{itemize}
\item {} 
we can read text from a file \texttt{f} using \texttt{text{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}open{\linebreak[0]}({\linebreak[0]}f{\linebreak[0]}){\linebreak[0]}.{\linebreak[0]}read{\linebreak[0]}({\linebreak[0]})}

\item {} 
we can read text from a URL \texttt{u} using \texttt{text{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}urlopen{\linebreak[0]}({\linebreak[0]}u{\linebreak[0]}){\linebreak[0]}.{\linebreak[0]}read{\linebreak[0]}({\linebreak[0]})}

\item {} 
NLTK comes with many corpora, e.g. the Brown Corpus, \texttt{corpus{\linebreak[0]}.{\linebreak[0]}brown}.

\item {} 
a word token is an individual occurrence of a word in a particular context

\item {} 
a word type is the vocabulary item, independent of any particular use of that item

\item {} 
tokenization is the segmentation of a text into basic units — or tokens —
such as words and punctuation.

\item {} 
tokenization based on whitespace is inadequate for many applications because it
bundles punctuation together with words

\item {} 
lemmatization is a process that maps the various forms of a word (such as \docutilsroleexample{appeared}, \docutilsroleexample{appears})
to the canonical or citation form of the word, also known as the lexeme or lemma (e.g. \docutilsrolelex{appear}).

\item {} 
a frequency distribution is a collection of items along with their frequency counts (e.g. the words of a
text and their frequency of appearance).

\item {} 
WordNet is a semantically-oriented dictionary of English, consisting of synonym sets — or synsets —
and organized into a hierarchical network.

\end{itemize}


%___________________________________________________________________________

\section{Further Reading}
\label{id24}

For a more extended treatment of regular expressions, see \href{\#appendix-regexps}{A}.
To learn about Unicode, see \href{\#appendix-unicode}{B}.

For more examples of processing words with NLTK, please see the
guides at \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}tokenize{\linebreak[0]}.{\linebreak[0]}html},
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}stem{\linebreak[0]}.{\linebreak[0]}html},
and \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}wordnet{\linebreak[0]}.{\linebreak[0]}html}.
A guide on accessing NLTK corpora is available at:
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}html}.
Chapters 2 and 3 of \cite{jurafskymartin2008} contain more advanced
material on regular expressions and morphology.
% John Hopkins Center for Language and Speech Processing, 1999
% Summer Workshop on Normalization of Non-Standard Words: Final Report
% http://www.clsp.jhu.edu/ws99/projects/normal/report.pdf 
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
\hypertarget{chap-tag}{}\setcounter{chapter}{2}

%___________________________________________________________________________

\chapter{Categorizing and Tagging Words}
\label{categorizing-and-tagging-words}
\label{chap-tag}
% TODO: exercise on cascaded tagging 
% TODO: simplified tagset (N, V, J, A) -- WordNet 
% TODO: better approach to tagging unknown words using special UNK token 
% TODO: motivate trigram tagging by showing some cases where bigram tagging doesn't work 
% TODO: xref to unicode section in prog chapter 
% TODO: * outstanding problems:
% - what are we doing with ConditionalFreqDist?
% - nltk.tag contains all of math library
% - nltk.corpus.brown.tagged_sents() is too verbose?
% - tag method returns generator object? 


%___________________________________________________________________________

\section{Introduction}
\label{id26}

In \href{\#chap-words}{Chapter 2} we dealt with words in their own right.
We looked at the distribution of \docutilsroleexample{often}, identifying the words that
follow it; we noticed that \docutilsroleexample{often} frequently modifies verbs.
In fact, it is a member of a whole class of verb-modifying words, the \emph{adverbs}.
Before we delve into this terminology, let's write a program that takes
a word and finds other words that appear in the same context (\href{\#dist-sim}{Listing 3.1}).
For example, given the word \docutilsroleexample{woman}, the program will find all
contexts where \docutilsroleexample{woman} appears in the corpus, such as \docutilsroleexample{the woman saw},
then searches for other words that appear in those contexts.

When we run \texttt{dist{\_}sim{\linebreak[0]}({\linebreak[0]})} on a few words, we find other words having
similar distribution:
searching for \docutilsroleexample{woman} finds \docutilsroleexample{man} and several other \emph{nouns};
searching for \docutilsroleexample{bought} finds \emph{verbs};
searching for \docutilsroleexample{over} finds \emph{prepositions};
searching for \docutilsroleexample{the} finds \emph{determiners}.
These labels — which may be familiar from grammar lessons — are
not just terms invented by grammarians, but labels for groups of words that
arise directly from the text.  These groups of words are so important that
they have several names, all in common use:
\index{word classes}\textbf{word classes}, \index{lexical categories}\textbf{lexical categories}, and \index{parts of speech}\textbf{parts of speech}.
We'll use these names interchangeably.
% .. doctest-ignore:: 
\hypertarget{dist-sim}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{build{\_}wc{\_}map}():
    \pysrcstring{"{}"{}"}
\pysrcstring{    Return a dictionary mapping words in the brown corpus to lists of}
\pysrcstring{    local lexical contexts, where a context is encoded as a tuple}
\pysrcstring{    (prevword, nextword).}
\pysrcstring{    "{}"{}"}
    wc{\_}map = nltk.defaultdict(list)
    words = {[}word.lower() \pysrckeyword{for} word \pysrckeyword{in} nltk.corpus.brown.words(){]}
    \pysrckeyword{for} i \pysrckeyword{in} range(1, len(words)-1):
        prevword, word, nextword = words{[}i-1:i+2{]}
        wc{\_}map{[}word{]}.append( (prevword, nextword) )
    return wc{\_}map

\pysrckeyword{def} \pysrcdefname{dist{\_}sim}(wc{\_}map, word, num=12):
    \pysrckeyword{if} word \pysrckeyword{in} wc{\_}map:
        contexts = set(wc{\_}map{[}word{]})
        fd = nltk.FreqDist(w \pysrckeyword{for} w \pysrckeyword{in} wc{\_}map \pysrckeyword{for} c \pysrckeyword{in} wc{\_}map{[}w{]} \pysrckeyword{if} c \pysrckeyword{in} contexts)
        return fd.sorted(){[}:num{]}
    return {[}{]}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }wc{\_}map = build{\_}wc{\_}map()
\pysrcprompt{>{}>{}> }dist{\_}sim(wc{\_}map, \pysrcstring{'woman'})
\pysrcoutput{{[}'man', 'number', 'woman', 'world', 'time', 'end', 'house', 'state',}
\pysrcoutput{ 'matter', 'kind', 'result', 'day'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }dist{\_}sim(wc{\_}map, \pysrcstring{'bought'})
\pysrcoutput{{[}'able', 'made', 'been', 'used', 'found', 'was', 'had', 'bought', ',',}
\pysrcoutput{ 'done', 'expected', 'given'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }dist{\_}sim(wc{\_}map, \pysrcstring{'over'})
\pysrcoutput{{[}'in', 'over', 'and', 'of', 'on', 'to', '.', ',', 'with', 'at', 'for', 'but'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }dist{\_}sim(wc{\_}map, \pysrcstring{'the'})
\pysrcoutput{{[}'the', 'a', 'his', 'this', 'and', 'in', 'their', 'an', 'her', 'that', 'no', 'its'{]}}}\end{alltt}
\caption{Program for Distributional Similarity}
\vspace{1ex}
\hrule
\end{pylisting}

One of the notable features of the Brown corpus is that all the words have
been \index{tagged}\textbf{tagged} for their part-of-speech. Now, instead of just
looking at the words that immediately follow \docutilsroleexample{often}, we can
look at the \index{part-of-speech tags}\textbf{part-of-speech tags} (or \index{POS tags}\textbf{POS tags}).  \href{\#often}{Table 3.1}
lists the top eight, ordered by frequency, along with explanations of
each tag.  As we can see, the majority of words following \docutilsroleexample{often} are verbs.
\hypertarget{often}{}
\leavevmode
\begin{longtable}[c]{|p{0.083\locallinewidth}|p{0.061\locallinewidth}|p{0.397\locallinewidth}|p{0.408\locallinewidth}|}
\hline
\textbf{
Tag
} & \textbf{
Freq
} & \textbf{
Example
} & \textbf{
Comment
} \\
\hline
\endhead

vbn
 & 
61
 & 
\docutilsroleexample{burnt}, \docutilsroleexample{gone}
 & 
verb: past participle
 \\
\hline

vb
 & 
51
 & 
\docutilsroleexample{make}, \docutilsroleexample{achieve}
 & 
verb: base form
 \\
\hline

vbd
 & 
36
 & 
\docutilsroleexample{saw}, \docutilsroleexample{looked}
 & 
verb: simple past tense
 \\
\hline

jj
 & 
30
 & 
\docutilsroleexample{ambiguous}, \docutilsroleexample{acceptable}
 & 
adjective
 \\
\hline

vbz
 & 
24
 & 
\docutilsroleexample{sees}, \docutilsroleexample{goes}
 & 
verb: third-person singular present
 \\
\hline

in
 & 
18
 & 
\docutilsroleexample{by}, \docutilsroleexample{in}
 & 
preposition
 \\
\hline

at
 & 
18
 & 
\docutilsroleexample{a}, \docutilsroleexample{this}
 & 
article
 \\
\hline

,
 & 
16
 & 
\docutilsroleexample{,}
 & 
comma
 \\
\hline
\caption{
Part of Speech Tags Following \docutilsroleexample{often} in the Brown Corpus
}\end{longtable}

The process of classifying words into their parts-of-speech and
labeling them accordingly is known as \index{part-of-speech tagging}\textbf{part-of-speech tagging},
\index{POS-tagging}\textbf{POS-tagging}, or simply \index{tagging}\textbf{tagging}.  The collection of tags
used for a particular task is known as a \index{tag set}\textbf{tag set}.  Our emphasis
in this chapter is on exploiting tags, and tagging text automatically.

Automatic tagging has several applications.  We have already
seen an example of how to exploit tags in corpus analysis — we
get a clear understanding of the distribution of \docutilsroleexample{often} by
looking at the tags of adjacent words.  Automatic tagging
also helps predict the behavior of previously unseen words.
For example, if we encounter the word
\docutilsroleexample{blogging} we can probably infer that it is a verb, with
the root \docutilsroleexample{blog}, and likely to occur after forms of the auxiliary
\texttt{to{\linebreak[0]} {\linebreak[0]}be} (e.g. \docutilsroleexample{he was blogging}).  Parts of speech are also used
in speech synthesis and recognition.  For example, \texttt{wind{\linebreak[0]}/{\linebreak[0]}NN}, as in
\docutilsroleexample{the wind blew}, is pronounced with a short vowel, whereas
\texttt{wind{\linebreak[0]}/{\linebreak[0]}VB}, as in \docutilsroleexample{to wind the clock}, is pronounced with a long
vowel.  Other examples can be found where the stress pattern differs
depending on whether the word is a noun or a verb, e.g. \docutilsroleexample{contest},
\docutilsroleexample{insult}, \docutilsroleexample{present}, \docutilsroleexample{protest}, \docutilsroleexample{rebel}, \docutilsroleexample{suspect}.
Without knowing the part of speech we cannot be sure of pronouncing
the word correctly.

In the next section we will see how to access and explore the Brown
Corpus.  Following this we will take a closer look at the
linguistics of word classes.  The rest of the chapter will deal with
automatic tagging: simple taggers, evaluation, and n-gram taggers.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Remember that our program samples assume you
begin your interactive session or your program with: \texttt{\pysrckeyword{import} {\linebreak[0]}nltk{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}re{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}pprint}
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\section{Getting Started with Tagging}
\label{getting-started-with-tagging}

Several large corpora, such as the Brown Corpus and portions of the
Wall Street Journal, have been tagged for part-of-speech, and we will be able to
process this tagged data.  Tagged corpus files typically contain text
of the following form (this example is from the Brown Corpus):
\begin{quote}{\ttfamily \raggedright \noindent
\small
The/at~grand/jj~jury/nn~commented/vbd~on/in~a/at~number/nn~of/in~\\
other/ap~topics/nns~,/,~among/in~them/ppo~the/at~Atlanta/np~and/cc~\\
Fulton/np-tl~County/nn-tl~purchasing/vbg~departments/nns~which/wdt~it/pps~\\
said/vbd~`{}`/`{}`~are/ber~well/ql~operated/vbn~and/cc~follow/vb~generally/rb~\\
accepted/vbn~practices/nns~which/wdt~inure/vb~to/in~the/at~best/jjt~\\
interest/nn~of/in~both/abx~governments/nns~'{}'/'{}'~./.
}\end{quote}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

The NLTK Brown Corpus reader converts part-of-speech tags to uppercase,
as this has become standard practice since the Brown Corpus was published.
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\subsection{Representing Tags and Reading Tagged Corpora}
\label{representing-tags-and-reading-tagged-corpora}

By convention in NLTK, a tagged token is represented using a Python
\index{tuple}\textbf{tuple}.
Python tuples are just like lists, except for one important
difference: tuples cannot be changed in place, for example by
\texttt{sort{\linebreak[0]}({\linebreak[0]})} or \texttt{reverse{\linebreak[0]}({\linebreak[0]})}. In other words, like strings, they are
immutable. Tuples are formed with the comma operator, and typically enclosed
using parentheses.  Like lists, tuples can be indexed and sliced:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }t = (\pysrcstring{'walk'}, \pysrcstring{'fem'}, 3)
\pysrcprompt{>{}>{}> }t{[}0{]}
\pysrcoutput{'walk'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }t{[}1:{]}
\pysrcoutput{('fem', 3)}
\pysrcoutput{}\pysrcprompt{>{}>{}> }t{[}0{]} = \pysrcstring{'run'}
\pysrcexcept{Traceback (most recent call last):}
\pysrcexcept{  File "<stdin>", line 1, in ?}
\pysrcexcept{TypeError: object does not support item assignment}}\end{alltt}

A tagged token is represented using a tuple consisting of just two items.
We can create one of these special tuples from the standard string
representation of a tagged token, using the function \texttt{str2tuple{\linebreak[0]}({\linebreak[0]})}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tagged{\_}token = nltk.tag.str2tuple(\pysrcstring{'fly/NN'})
\pysrcprompt{>{}>{}> }tagged{\_}token
\pysrcoutput{('fly', 'NN')}
\pysrcoutput{}\pysrcprompt{>{}>{}> }tagged{\_}token{[}0{]}
\pysrcoutput{'fly'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }tagged{\_}token{[}1{]}
\pysrcoutput{'NN'}}\end{alltt}

We can construct a list of tagged tokens directly from a string.  The first
step is to tokenize the string
to access the individual \texttt{word{\linebreak[0]}/{\linebreak[0]}tag} strings, and then to convert
each of these into a tuple (using \texttt{str2tuple{\linebreak[0]}({\linebreak[0]})}).  We do this in
two ways.  The first method, starting at line \href{\#str2tuple1}{①}, initializes
an empty list \texttt{tagged{\_}words}, loops over the \texttt{word{\linebreak[0]}/{\linebreak[0]}tag} tokens, converts them into
tuples, appends them to \texttt{tagged{\_}words}, and finally displays the result.
The second method, on line \href{\#str2tuple2}{②}, uses a list comprehension to do
the same work in a way that is not only more compact, but also more readable.
(List comprehensions were introduced in \href{\#sec-list-comprehensions}{section 2.3.3}).
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = \pysrcstring{'{}'{}'}
\pysrcmore{... }\pysrcstring{The/AT grand/JJ jury/NN commented/VBD on/IN a/AT number/NN of/IN}
\pysrcmore{... }\pysrcstring{other/AP topics/NNS ,/, AMONG/IN them/PPO the/AT Atlanta/NP and/CC}
\pysrcmore{... }\pysrcstring{Fulton/NP-tl County/NN-tl purchasing/VBG departments/NNS which/WDT it/PPS}
\pysrcmore{... }\pysrcstring{said/VBD `{}`/`{}` ARE/BER well/QL operated/VBN and/CC follow/VB generally/RB}
\pysrcmore{... }\pysrcstring{accepted/VBN practices/NNS which/WDT inure/VB to/IN the/AT best/JJT}
\pysrcmore{... }\pysrcstring{interest/NN of/IN both/ABX governments/NNS '{}'/'{}' ./.}
\pysrcmore{... }\pysrcstring{'{}'{}'}
\pysrcprompt{>{}>{}> }tagged{\_}words = {[}{]}                                   ①
\pysrcprompt{>{}>{}> }\pysrckeyword{for} t \pysrckeyword{in} sent.split():
\pysrcmore{... }    tagged{\_}words.append(nltk.tag.str2tuple(t))
\pysrcprompt{>{}>{}> }tagged{\_}words
\pysrcoutput{{[}('The', 'AT'), ('grand', 'JJ'), ('jury', 'NN'), ('commented', 'VBD'),}
\pysrcoutput{('on', 'IN'), ('a', 'AT'), ('number', 'NN'), ... ('.', '.'){]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }{[}nltk.tag.str2tuple(t) \pysrckeyword{for} t \pysrckeyword{in} sent.split(){]}   ②
\pysrcoutput{{[}('The', 'AT'), ('grand', 'JJ'), ('jury', 'NN'), ('commented', 'VBD'),}
\pysrcoutput{('on', 'IN'), ('a', 'AT'), ('number', 'NN'), ... ('.', '.'){]}}}\end{alltt}

We can access several tagged corpora directly from Python.
If a corpus contains tagged text, then it will have a \texttt{tagged{\_}words{\linebreak[0]}({\linebreak[0]})}
method.  Please see the \texttt{README} file included with each corpus
for documentation of its tagset.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.corpus.brown.tagged{\_}words()
\pysrcoutput{{[}('The', 'AT'), ('Fulton', 'NP-TL'), ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.corpus.nps{\_}chat.tagged{\_}words()
\pysrcoutput{{[}('now', 'RB'), ('im', 'PRP'), ('left', 'VBD'), ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.conll2000.tagged{\_}words()
\pysrcoutput{{[}('Confidence', 'NN'), ('in', 'IN'), ('the', 'DT'), ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.treebank.tagged{\_}words()
\pysrcoutput{{[}('Pierre', 'NNP'), ('Vinken', 'NNP'), (',', ','), ...{]}}}\end{alltt}

Tagged corpora for several other languages are distributed with NLTK,
including Chinese, Hindi, Portuguese, Spanish, Dutch and Catalan.
These usually contain non-ASCII text,
and Python always displays this in hexadecimal when printing a larger structure
such as a list.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.corpus.sinica{\_}treebank.tagged{\_}words()
\pysrcoutput{{[}('{\textbackslash}xe4{\textbackslash}xb8{\textbackslash}x80', 'Neu'), ('{\textbackslash}xe5{\textbackslash}x8f{\textbackslash}x8b{\textbackslash}xe6{\textbackslash}x83{\textbackslash}x85', 'Nad'), ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.indian.tagged{\_}words()
\pysrcoutput{{[}('{\textbackslash}xe0{\textbackslash}xa6{\textbackslash}xae{\textbackslash}xe0{\textbackslash}xa6{\textbackslash}xb9{\textbackslash}xe0{\textbackslash}xa6{\textbackslash}xbf{\textbackslash}xe0{\textbackslash}xa6{\textbackslash}xb7{\textbackslash}xe0{\textbackslash}xa7{\textbackslash}x87{\textbackslash}xe0{\textbackslash}xa6{\textbackslash}xb0', 'NN'),}
\pysrcoutput{('{\textbackslash}xe0{\textbackslash}xa6{\textbackslash}xb8{\textbackslash}xe0{\textbackslash}xa6{\textbackslash}xa8{\textbackslash}xe0{\textbackslash}xa7{\textbackslash}x8d{\textbackslash}xe0{\textbackslash}xa6{\textbackslash}xa4{\textbackslash}xe0{\textbackslash}xa6{\textbackslash}xbe{\textbackslash}xe0{\textbackslash}xa6{\textbackslash}xa8', 'NN'), ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.mac{\_}morpho.tagged{\_}words()
\pysrcoutput{{[}('Jersei', 'N'), ('atinge', 'V'), ('m{\textbackslash}xe9dia', 'N'), ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.conll2002.tagged{\_}words()
\pysrcoutput{{[}('Sao', 'NC'), ('Paulo', 'VMI'), ('(', 'Fpa'), ...{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.cess{\_}cat.tagged{\_}words()
\pysrcoutput{{[}('El', 'da0ms0'), ('Tribunal{\_}Suprem', 'np0000o'), ...{]}}}\end{alltt}

If your environment is set up correctly, with appropriate editors and fonts,
you should be able to display individual strings in a human-readable way.
For example, \href{\#tag-indian}{Figure 3.1} shows the output of the demonstration code
(\texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}indian{\linebreak[0]}.{\linebreak[0]}demo{\linebreak[0]}({\linebreak[0]})}).
\hypertarget{tag-indian}{}\begin{figure}[htbp]
\centering

\scalebox{0.308571}{\includegraphics{../images/tag-indian.png}}
\caption{POS-Tagged Data from Four Indian Languages}\end{figure}
% इराक_NNP के_PREP विदेश_NNC मंत्री_NN ने_PREP अमरीका_NNP के_PREP उस_PRP प्रस्ताव_NN का_PREP मजाक_NVB उड़ाया_VFM है_VAUX ... 

If the corpus is also segmented into sentences, it will have
a \texttt{tagged{\_}sents{\linebreak[0]}({\linebreak[0]})} method that returns a list of tagged sentences.
This will be useful when we come to training automatic taggers,
as they typically function on a sentence at a time.


%___________________________________________________________________________

\subsection{Nouns and Verbs}
\label{nouns-and-verbs}

Linguists recognize several major categories of words in
English, such as nouns, verbs, adjectives and determiners.  In this
section we will discuss the most important categories, namely nouns
and verbs.

Nouns generally refer to people, places, things, or concepts, e.g.:
\docutilsroleexample{woman, Scotland, book, intelligence}.  Nouns can appear after
determiners and adjectives, and can be the subject or object of the
verb, as shown in \href{\#syntax-nouns}{Table 3.2}.
\hypertarget{syntax-nouns}{}
\leavevmode
\begin{longtable}[c]{|p{0.115\locallinewidth}|p{0.394\locallinewidth}|p{0.436\locallinewidth}|}
\hline
\textbf{
Word
} & \textbf{
After a determiner
} & \textbf{
Subject of the verb
} \\
\hline
\endhead

woman
 & 
\emph{the} woman who I saw yesterday ...
 & 
the woman \emph{sat} down
 \\
\hline

Scotland
 & 
\emph{the} Scotland I remember as a child ...
 & 
Scotland \emph{has} five million people
 \\
\hline

book
 & 
\emph{the} book I bought yesterday ...
 & 
this book \emph{recounts} the colonization of Australia
 \\
\hline

intelligence
 & 
\emph{the} intelligence displayed by the child ...
 & 
Mary's intelligence \emph{impressed} her teachers
 \\
\hline
\caption{
Syntactic Patterns involving some Nouns
}\end{longtable}

Nouns can be classified as \index{common nouns}\textbf{common nouns} and \index{proper nouns}\textbf{proper nouns}.
Proper nouns identify particular individuals or entities,
e.g. \docutilsroleexample{Moses} and \docutilsroleexample{Scotland}.  Common nouns are all the rest.
Another distinction exists between \index{count nouns}\textbf{count nouns} and \index{mass
nouns}\textbf{mass
nouns}. Count nouns are thought of as distinct entities that can
be counted, such as \docutilsroleexample{pig} (e.g. \docutilsroleexample{one pig, two pigs, many
pigs}).  They cannot occur with the word \docutilsroleexample{much} (i.e. *\docutilsroleexample{much pigs}).  Mass nouns, on the other hand, are not thought of
as distinct entities (e.g. \docutilsroleexample{sand}).  They cannot be pluralized,
and do not occur with numbers (e.g. *\docutilsroleexample{two sands}, *\docutilsroleexample{many
sands}).  However, they can occur with \docutilsroleexample{much} (i.e. \docutilsroleexample{much
sand}).

Verbs are words that describe events and actions, e.g. \docutilsroleexample{fall},
\docutilsroleexample{eat} in \href{\#syntax-verbs}{Table 3.3}.
In the context of a sentence, verbs express a relation
involving the referents of one or more noun phrases.
\hypertarget{syntax-verbs}{}
\leavevmode
\begin{longtable}[c]{|p{0.075\locallinewidth}|p{0.191\locallinewidth}|p{0.563\locallinewidth}|}
\hline
\textbf{
Word
} & \textbf{
Simple
} & \textbf{
With modifiers and adjuncts (italicized)
} \\
\hline
\endhead

fall
 & 
Rome fell
 & 
Dot com stocks \emph{suddenly} fell \emph{like a stone}
 \\
\hline

eat
 & 
Mice eat cheese
 & 
John ate the pizza \emph{with gusto}
 \\
\hline
\caption{
Syntactic Patterns involving some Verbs
}\end{longtable}

\pagebreak

Verbs can be classified according to the number of arguments (usually
noun phrases) that they require.  The word \docutilsroleexample{fall} is
\index{intransitive}\textbf{intransitive}, requiring exactly one argument (the entity that
falls).  The word \docutilsroleexample{eat} is \index{transitive}\textbf{transitive}, requiring two
arguments (the eater and the eaten).  Other verbs are more complex;
for instance \docutilsroleexample{put} requires three arguments, the agent doing the
putting, the entity being put somewhere, and a location.  We will return
to this topic when we come to look at grammars and parsing (see \href{\#chap-parse}{Chapter 7}).

In the Brown Corpus, verbs have a range of possible tags, e.g.:
\texttt{give{\linebreak[0]}/{\linebreak[0]}VB} (present), \texttt{gives{\linebreak[0]}/{\linebreak[0]}VBZ} (present, 3ps),
\texttt{giving{\linebreak[0]}/{\linebreak[0]}VBG} (present continuous; gerund)
\texttt{gave{\linebreak[0]}/{\linebreak[0]}VBD} (simple past),
and \texttt{given{\linebreak[0]}/{\linebreak[0]}VBN} (past participle).
We will discuss these tags in more detail in a later section.


%___________________________________________________________________________

\subsection{Nouns and Verbs in Tagged Corpora}
\label{nouns-and-verbs-in-tagged-corpora}

Now that we are able to access tagged corpora, we can write simple
programs to garner statistics about the tags.  In this section
we will focus on the nouns and verbs.

What are the 10 most common verbs?  We can write a program to find all
words tagged with \docutilsrolecategory{VB}, \docutilsrolecategory{VBZ}, \docutilsrolecategory{VBG}, \docutilsrolecategory{VBD} or \docutilsrolecategory{VBN}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fd = nltk.FreqDist()
\pysrcprompt{>{}>{}> }\pysrckeyword{for} (wd, tg) \pysrckeyword{in} nltk.corpus.brown.tagged{\_}words(categories=\pysrcstring{'a'}):
\pysrcmore{... }    \pysrckeyword{if} tg{[}:2{]} == \pysrcstring{'VB'}:
\pysrcmore{... }        fd.inc(wd + \pysrcstring{"/"} + tg)
\pysrcprompt{>{}>{}> }fd.sorted(){[}:20{]}
\pysrcoutput{{[}'said/VBD', 'get/VB', 'made/VBN', 'United/VBN-TL', 'take/VB',}
\pysrcoutput{ 'took/VBD', 'told/VBD', 'made/VBD', 'make/VB', 'got/VBD',}
\pysrcoutput{ 'came/VBD', 'go/VB', 'see/VB', 'went/VBD', 'given/VBN',}
\pysrcoutput{ 'expected/VBN', 'began/VBD', 'give/VB', 'taken/VBN', 'play/VB'{]}}}\end{alltt}

Let's study nouns, and find the most frequent nouns of each noun
part-of-speech type.  The program in \href{\#findtags}{Listing 3.2} finds all
tags starting with \docutilsrolecategory{NN}, and provides a few example words for each
one.  Observe that there are many noun tags; the most important
of these have \texttt{{\$}} for possessive nouns, \docutilsrolecategory{S} for plural nouns (since plural
nouns typically end in \docutilsroleexample{s}), \docutilsrolecategory{P} for proper nouns.
\hypertarget{findtags}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{findtags}(tag{\_}prefix, tagged{\_}text):
    cfd = nltk.ConditionalFreqDist()
    \pysrckeyword{for} (wd, tg) \pysrckeyword{in} tagged{\_}text:
        \pysrckeyword{if} tg.startswith(tag{\_}prefix):
            cfd{[}tg{]}.inc(wd)
    tagdict = {\{}{\}}
    \pysrckeyword{for} tg \pysrckeyword{in} cfd.conditions():
        tagdict{[}tg{]} = cfd{[}tg{]}.sorted(){[}:5{]}
    return tagdict}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tagdict = findtags(\pysrcstring{'NN'}, nltk.corpus.brown.tagged{\_}words(categories=\pysrcstring{'a'}))
\pysrcprompt{>{}>{}> }\pysrckeyword{for} tg \pysrckeyword{in} sorted(tagdict):
\pysrcmore{... }    \pysrckeyword{print} tg, tagdict{[}tg{]}
\pysrcoutput{NN {[}'year', 'time', 'state', 'week', 'man'{]}}
\pysrcoutput{NN{\$} {[}"year's", "world's", "state's", "nation's", "company's"{]}}
\pysrcoutput{NN{\$}-HL {[}"Golf's", "Navy's"{]}}
\pysrcoutput{NN{\$}-TL {[}"President's", "University's", "League's", "Gallery's", "Army's"{]}}
\pysrcoutput{NN-HL {[}'cut', 'Salary', 'condition', 'Question', 'business'{]}}
\pysrcoutput{NN-NC {[}'eva', 'ova', 'aya'{]}}
\pysrcoutput{NN-TL {[}'President', 'House', 'State', 'University', 'City'{]}}
\pysrcoutput{NN-TL-HL {[}'Fort', 'City', 'Commissioner', 'Grove', 'House'{]}}
\pysrcoutput{NNS {[}'years', 'members', 'people', 'sales', 'men'{]}}
\pysrcoutput{NNS{\$} {[}"children's", "women's", "men's", "janitors'", "taxpayers'"{]}}
\pysrcoutput{NNS{\$}-HL {[}"Dealers'", "Idols'"{]}}
\pysrcoutput{NNS{\$}-TL {[}"Women's", "States'", "Giants'", "Officers'", "Bombers'"{]}}
\pysrcoutput{NNS-HL {[}'years', 'idols', 'Creations', 'thanks', 'centers'{]}}
\pysrcoutput{NNS-TL {[}'States', 'Nations', 'Masters', 'Rules', 'Communists'{]}}
\pysrcoutput{NNS-TL-HL {[}'Nations'{]}}}\end{alltt}
\caption{Program to Find the Most Frequent Noun Tags}
\vspace{1ex}
\hrule
\end{pylisting}

Some tags contain a plus sign; these are compound tags, and are
assigned to words that contain two parts normally treated separately.
Some tags contain a minus sign; this indicates disjunction.


%___________________________________________________________________________

\subsection{The Default Tagger}
\label{the-default-tagger}

The simplest possible tagger assigns the same tag to each token.  This
may seem to be a rather banal step, but it establishes an important
baseline for tagger performance.  In order to get the best result, we
tag each word with the most likely word.  (This kind of tagger is
known as a \index{majority class classifier}\textbf{majority class classifier}).  What then, is the most
frequent tag?  We can find out using a simple program:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fd = nltk.FreqDist()
\pysrcprompt{>{}>{}> }\pysrckeyword{for} (wd, tg) \pysrckeyword{in} nltk.corpus.brown.tagged{\_}words(categories=\pysrcstring{'a'}):
\pysrcmore{... }    fd.inc(tg)
\pysrcprompt{>{}>{}> }fd.max()
\pysrcoutput{'NN'}}\end{alltt}

Now we can create a tagger, called \texttt{default{\_}tagger}, that
tags everything as \docutilsrolecategory{NN}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tokens = \pysrcstring{'John saw 3 polar bears .'}.split()
\pysrcprompt{>{}>{}> }default{\_}tagger = nltk.DefaultTagger(\pysrcstring{'NN'})
\pysrcprompt{>{}>{}> }default{\_}tagger.tag(tokens)
\pysrcoutput{{[}('John', 'NN'), ('saw', 'NN'), ('3', 'NN'), ('polar', 'NN'),}
\pysrcoutput{('bears', 'NN'), ('.', 'NN'){]}}}\end{alltt}

This is a simple algorithm, and it performs poorly when used on its
own. On a typical corpus, it will tag only about an eighth of the tokens
correctly:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.tag.accuracy(default{\_}tagger, nltk.corpus.brown.tagged{\_}sents(categories=\pysrcstring{'a'}))
\pysrcoutput{0.13089484257215028}}\end{alltt}

Default taggers assign their tag to every single word, even words that
have never been encountered before.  As it happens, most new words are
nouns.  Thus, default taggers help to improve the
robustness of a language processing system.  We will return to them
later, in the context of our discussion of \emph{backoff}.


%___________________________________________________________________________

\subsection{Exercises}
\label{id27}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼
Working with someone else, take turns to pick a word that can be
either a noun or a verb (e.g. \docutilsroleexample{contest}); the opponent has to
predict which one is likely to be the most frequent in the Brown corpus; check the
opponent's prediction, and tally the score over several turns.

\item {} 
◑ Write programs to process the Brown Corpus and find answers to the following
questions:
\setcounter{listcnt1}{0}
\begin{list}{\arabic{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Which nouns are more common in their plural form, rather than their singular
form? (Only consider regular plurals, formed with the \docutilsroleexample{-s} suffix.)

\item {} 
Which word has the greatest number of distinct tags.  What are they, and
what do they represent?

\item {} 
List tags in order of decreasing frequency.  What do the 20 most frequent tags represent?

\item {} 
Which tags are nouns most commonly found after?  What do these tags represent?

\end{list}

\item {} 
◑ Generate some statistics for tagged data to answer the following questions:
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
What proportion of word types are always assigned the same part-of-speech tag?

\item {} 
How many words are ambiguous, in the sense that they appear with at least two tags?

\item {} 
What percentage of word \emph{occurrences} in the Brown Corpus involve
these ambiguous words?

\end{list}

\item {} 
◑ Above we gave an example of the \texttt{nltk{\linebreak[0]}.{\linebreak[0]}tag{\linebreak[0]}.{\linebreak[0]}accuracy{\linebreak[0]}({\linebreak[0]})} function.  It has two
arguments, a tagger and some tagged text, and it works out how accurately
the tagger performs on this text.  For example, if the supplied tagged text
was \texttt{{[}{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}the{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}DT{\linebreak[0]}'}){\linebreak[0]},{\linebreak[0]} {\linebreak[0]}(\pysrcstring{'{\linebreak[0]}dog{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}NN{\linebreak[0]}'}){\linebreak[0]}{]}} and the tagger produced the output
\texttt{{[}{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}the{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}NN{\linebreak[0]}'}){\linebreak[0]},{\linebreak[0]} {\linebreak[0]}(\pysrcstring{'{\linebreak[0]}dog{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}NN{\linebreak[0]}'}){\linebreak[0]}{]}}, then the accuracy score would be \texttt{0{\linebreak[0]}.{\linebreak[0]}5}.
Can you figure out how the \texttt{nltk{\linebreak[0]}.{\linebreak[0]}tag{\linebreak[0]}.{\linebreak[0]}accuracy{\linebreak[0]}({\linebreak[0]})} function works?
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
A tagger takes a list of words as input, and produces a list of tagged words
as output.  However, \texttt{nltk{\linebreak[0]}.{\linebreak[0]}tag{\linebreak[0]}.{\linebreak[0]}accuracy{\linebreak[0]}({\linebreak[0]})} is given correctly tagged text as its input.
What must the \texttt{nltk{\linebreak[0]}.{\linebreak[0]}tag{\linebreak[0]}.{\linebreak[0]}accuracy{\linebreak[0]}({\linebreak[0]})} function do with this input before performing the tagging?

\item {} 
Once the supplied tagger has created newly tagged text, how would \texttt{nltk{\linebreak[0]}.{\linebreak[0]}tag{\linebreak[0]}.{\linebreak[0]}accuracy{\linebreak[0]}({\linebreak[0]})}
go about comparing it with the original tagged text and computing the accuracy score?

\end{list}

\end{list}


%___________________________________________________________________________

\section{Looking for Patterns in Words}
\label{looking-for-patterns-in-words}


%___________________________________________________________________________

\subsection{Some Morphology}
\label{some-morphology}

English nouns can be morphologically complex.  For example, words like
\docutilsroleexample{books} and \docutilsroleexample{women} are plural.  Words with the \docutilsroleexample{-ness}
suffix are nouns that have been derived from adjectives,
e.g. \docutilsroleexample{happiness} and \docutilsroleexample{illness}.  The \docutilsroleexample{-ment} suffix
appears on certain nouns derived from verbs, e.g. \docutilsroleexample{government} and
\docutilsroleexample{establishment}.

English verbs can also be morphologically complex.  For instance, the
\index{present participle}\textbf{present participle} of a verb ends in \docutilsroleexample{-ing}, and expresses
the idea of ongoing, incomplete action (e.g. \docutilsroleexample{falling}, \docutilsroleexample{eating}).
The \docutilsroleexample{-ing} suffix also appears on nouns derived from verbs, e.g. \docutilsroleexample{the
falling of the leaves} (this is known as the \index{gerund}\textbf{gerund}).
In the Brown corpus, these are tagged \docutilsrolecategory{VBG}.

The \index{past participle}\textbf{past participle} of a verb often ends in \docutilsroleexample{-ed}, and
expresses the idea of a completed action (e.g. \docutilsroleexample{walked}, \docutilsroleexample{cried}).
These are tagged \docutilsrolecategory{VBD}.
% TODO: Modal verbs, e.g. `would`:lx: ... 

Common tag sets often capture some \index{morpho-syntactic}\textbf{morpho-syntactic} information;
that is, information about the kind of morphological markings that
words receive by virtue of their syntactic role.  Consider, for
example, the selection of distinct grammatical forms of the word
\docutilsroleexample{go} illustrated in the following sentences:
\hypertarget{go}{}\begin{itemize}
\item[(4)] \begin{itemize}
\item[a.] 
\emph{Go} away!
\end{itemize}
\begin{itemize}
\item[b.] 
He sometimes \emph{goes} to the cafe.
\end{itemize}
\begin{itemize}
\item[c.] 
All the cakes have \emph{gone}.
\end{itemize}
\begin{itemize}
\item[d.] 
We \emph{went} on the excursion.
\end{itemize}
\end{itemize}

Each of these forms — \docutilsroleexample{go}, \docutilsroleexample{goes}, \docutilsroleexample{gone}, and \docutilsroleexample{went} —
is morphologically distinct from the others. Consider the form,
\docutilsroleexample{goes}. This cannot occur in all grammatical contexts, but
requires, for instance, a third person singular subject. Thus, the
following sentences are ungrammatical.
\begin{itemize}
\item[(5)] \begin{itemize}
\item[a.] 
*They sometimes \emph{goes} to the cafe.
\end{itemize}
\begin{itemize}
\item[b.] 
*I sometimes \emph{goes} to the cafe.
\end{itemize}
\end{itemize}

By contrast, \docutilsroleexample{gone} is the past participle form; it is required
after \docutilsroleexample{have} (and cannot be replaced in this context by
\docutilsroleexample{goes}), and cannot occur as the main verb of a clause.
\begin{itemize}
\item[(6)] \begin{itemize}
\item[a.] 
*All the cakes have \emph{goes}.
\end{itemize}
\begin{itemize}
\item[b.] 
*He sometimes \emph{gone} to the cafe.
\end{itemize}
\end{itemize}

We can easily imagine a tag set in which the four distinct
grammatical forms just discussed were all tagged as \docutilsrolecategory{VB}. Although
this would be adequate for some purposes, a more fine-grained tag set
will provide useful information about these forms that can be of value
to other processors that try to detect syntactic patterns from tag
sequences. As we noted at the beginning of this chapter,
the Brown tag set does in fact capture these distinctions,
as summarized in \href{\#morphosyntax}{Table 3.4}.
\hypertarget{morphosyntax}{}
\leavevmode
\begin{longtable}[c]{|p{0.098\locallinewidth}|p{0.272\locallinewidth}|p{0.075\locallinewidth}|}
\hline
\textbf{
Form
} & \textbf{
Category
} & \textbf{
Tag
} \\
\hline
\endhead

go
 & 
base
 & 
VB
 \\
\hline

goes
 & 
3rd singular present
 & 
VBZ
 \\
\hline

gone
 & 
past participle
 & 
VBN
 \\
\hline

going
 & 
gerund
 & 
VBG
 \\
\hline

went
 & 
simple past
 & 
VBD
 \\
\hline
\caption{
Some morphosyntactic distinctions in the Brown tag set
}\end{longtable}

In addition to this set of verb tags, the various forms of the verb \docutilsroleexample{to be}
have special tags:
\texttt{be{\linebreak[0]}/{\linebreak[0]}BE{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}being{\linebreak[0]}/{\linebreak[0]}BEG{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}am{\linebreak[0]}/{\linebreak[0]}BEM{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}been{\linebreak[0]}/{\linebreak[0]}BEN} and \texttt{was{\linebreak[0]}/{\linebreak[0]}BEDZ}.   All told,
this fine-grained tagging of verbs means that an automatic tagger
that uses this tag set is in effect carrying out a limited amount
of morphological analysis.

Most part-of-speech tag sets make use of the same basic categories,
such as noun, verb, adjective, and preposition. However, tag sets
differ both in how finely they divide words into categories, and in
how they define their categories. For example, \docutilsroleexample{is} might be tagged
simply as a verb in one tag set; but as a distinct form of the lexeme \docutilsroleexample{BE}
in another tag set (as in the Brown Corpus).  This variation in tag sets is
unavoidable, since part-of-speech tags are used in different ways for
different tasks. In other words, there is no one 'right way' to assign
tags, only more or less useful ways depending on one's goals.
More details about the Brown corpus tag set can be found in the \href{\#appendix}{Appendix}
at the end of this chapter.
\hypertarget{sec-regular-expression-tagger}{}

%___________________________________________________________________________

\subsection{The Regular Expression Tagger}
\label{the-regular-expression-tagger}
\label{sec-regular-expression-tagger}

The regular expression tagger assigns tags to tokens on the basis of
matching patterns.  For instance, we might guess that any word ending
in \docutilsroleexample{ed} is the past participle of a verb, and any word ending with
\docutilsroleexample{'s} is a possessive noun.  We can express these as a list of
regular expressions:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }patterns = {[}
\pysrcmore{... }    (r\pysrcstring{'.*ing{\$}'}, \pysrcstring{'VBG'}),               \pysrccomment{{\#} gerunds}
\pysrcmore{... }    (r\pysrcstring{'.*ed{\$}'}, \pysrcstring{'VBD'}),                \pysrccomment{{\#} simple past}
\pysrcmore{... }    (r\pysrcstring{'.*es{\$}'}, \pysrcstring{'VBZ'}),                \pysrccomment{{\#} 3rd singular present}
\pysrcmore{... }    (r\pysrcstring{'.*ould{\$}'}, \pysrcstring{'MD'}),               \pysrccomment{{\#} modals}
\pysrcmore{... }    (r\pysrcstring{'.*{\textbackslash}'s{\$}'}, \pysrcstring{'NN{\$}'}),               \pysrccomment{{\#} possessive nouns}
\pysrcmore{... }    (r\pysrcstring{'.*s{\$}'}, \pysrcstring{'NNS'}),                 \pysrccomment{{\#} plural nouns}
\pysrcmore{... }    (r\pysrcstring{'{\textasciicircum}-?{[}0-9{]}+(.{[}0-9{]}+)?{\$}'}, \pysrcstring{'CD'}),  \pysrccomment{{\#} cardinal numbers}
\pysrcmore{... }    (r\pysrcstring{'.*'}, \pysrcstring{'NN'})                     \pysrccomment{{\#} nouns (default)}
\pysrcmore{... }{]}}\end{alltt}

Note that these are processed in order, and the first one that matches is applied.

Now we can set up a tagger and use it to tag some text.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }regexp{\_}tagger = nltk.RegexpTagger(patterns)
\pysrcprompt{>{}>{}> }regexp{\_}tagger.tag(nltk.corpus.brown.sents(categories=\pysrcstring{'a'}){[}3{]})
\pysrcoutput{{[}('`{}`', 'NN'), ('Only', 'NN'), ('a', 'NN'), ('relative', 'NN'),}
\pysrcoutput{('handful', 'NN'), ('of', 'NN'), ('such', 'NN'), ('reports', 'NNS'),}
\pysrcoutput{('was', 'NNS'), ('received', 'VBD'), ("'{}'", 'NN'), (',', 'NN'),}
\pysrcoutput{('the', 'NN'), ('jury', 'NN'), ('said', 'NN'), (',', 'NN'), ('`{}`', 'NN'),}
\pysrcoutput{('considering', 'VBG'), ('the', 'NN'), ('widespread', 'NN'), ..., ('.', 'NN'){]}}}\end{alltt}

How well does this do?
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.tag.accuracy(regexp{\_}tagger, nltk.corpus.brown.tagged{\_}sents(categories=\pysrcstring{'a'}))
\pysrcoutput{0.20326391789486245}}\end{alltt}

The regular expression is a catch-all that tags everything as a noun.
This is equivalent to the default tagger (only much less efficient).
Instead of re-specifying this as part of the regular expression tagger,
is there a way to combine this tagger with the default tagger?  We
will see how to do this later, under the heading of backoff taggers.


%___________________________________________________________________________

\subsection{Exercises}
\label{id28}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼
Search the web for ``spoof newspaper headlines'', to find such gems as:
\docutilsroleexample{British Left Waffles on Falkland Islands}, and
\docutilsroleexample{Juvenile Court to Try Shooting Defendant}.
Manually tag these headlines to see if knowledge of the part-of-speech
tags removes the ambiguity.

\item {} 
☼ Satisfy yourself that there are
restrictions on the distribution of \docutilsroleexample{go} and \docutilsroleexample{went}, in the
sense that they cannot be freely interchanged in the kinds of contexts
illustrated in \href{\#go}{(4d)}.

\item {} 
◑ Write code to search the Brown Corpus for particular words and phrases
according to tags, to answer the following questions:
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Produce an alphabetically sorted list of the distinct words tagged as \texttt{MD}.

\item {} 
Identify words that can be plural nouns or third person singular verbs
(e.g. \docutilsroleexample{deals}, \docutilsroleexample{flies}).

\item {} 
Identify three-word prepositional phrases of the form IN + DET + NN
(eg. \docutilsroleexample{in the lab}).

\item {} 
What is the ratio of masculine to feminine pronouns?

\end{list}

\item {} 
◑ In the introduction we saw a table involving frequency counts for
the verbs \docutilsroleexample{adore}, \docutilsroleexample{love}, \docutilsroleexample{like}, \docutilsroleexample{prefer} and
preceding qualifiers such as \docutilsroleexample{really}.  Investigate the full
range of qualifiers (Brown tag \docutilsrolecategory{QL}) that appear before these
four verbs.

\item {} 
◑
We defined the \texttt{regexp{\_}tagger} that can be used
as a fall-back tagger for unknown words.  This tagger only checks for
cardinal numbers.  By testing for particular prefix or suffix strings,
it should be possible to guess other tags.  For example,
we could tag any word that ends with \docutilsroleexample{-s} as a plural noun.
Define a regular expression tagger (using \texttt{nltk{\linebreak[0]}.{\linebreak[0]}RegexpTagger})
that tests for at least five other patterns in the spelling of words.
(Use inline documentation to explain the rules.)

\item {} 
◑
Consider the regular expression tagger developed in the exercises in
the previous section.  Evaluate the tagger using \texttt{nltk{\linebreak[0]}.{\linebreak[0]}tag{\linebreak[0]}.{\linebreak[0]}accuracy{\linebreak[0]}({\linebreak[0]})},
and try to come up with ways to improve its performance.  Discuss your findings.
How does objective evaluation help in the development process?

\item {} 
★ There are 264 distinct words in the Brown Corpus having exactly
three possible tags.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Print a table with the integers 1..10 in one column, and the
number of distinct words in the corpus having 1..10 distinct tags.

\item {} 
For the word with the greatest number of distinct tags, print
out sentences from the corpus containing the word, one for each
possible tag.

\end{list}

\item {} 
★ Write a program to classify contexts involving the word \docutilsroleexample{must} according
to the tag of the following word.  Can this be used to discriminate between the
epistemic and deontic uses of \docutilsroleexample{must}?

\end{list}


%___________________________________________________________________________

\section{Baselines and Backoff}
\label{baselines-and-backoff}

So far the performance of our simple taggers has been disappointing.
Before we embark on a process to get 90+{\%} performance, we need to do
two more things.  First, we need to establish a more principled
baseline performance than the default tagger, which was too
simplistic, and the regular expression tagger, which was too
arbitrary.  Second, we need a way to connect multiple taggers
together, so that if a more specialized tagger is unable to assign
a tag, we can ``back off'' to a more generalized tagger.
\hypertarget{sec-lookup-tagger}{}

%___________________________________________________________________________

\subsection{The Lookup Tagger}
\label{the-lookup-tagger}
\label{sec-lookup-tagger}

A lot of high-frequency words do not have the \texttt{NN} tag.
Let's find some of these words and their tags.  The
following code takes a list of sentences and counts up the words,
and prints the 100 most frequent words:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fd = nltk.FreqDist(nltk.corpus.brown.words(categories=\pysrcstring{'a'}))
\pysrcprompt{>{}>{}> }most{\_}freq{\_}words = fd.sorted(){[}:100{]}
\pysrcprompt{>{}>{}> }most{\_}freq{\_}words
\pysrcoutput{{[}'the', ',', '.', 'of', 'and', 'to', 'a', 'in', 'for', 'The', 'that', '`{}`',}
\pysrcoutput{'is', 'was', "'{}'", 'on', 'at', 'with', 'be', 'by', 'as', 'he', 'said', 'his',}
\pysrcoutput{'will', 'it', 'from', 'are', ';', 'has', 'an', '-{}-', 'had', 'who', 'have',}
\pysrcoutput{'not', 'Mrs.', 'were', 'this', 'would', 'which', 'their', 'been', 'they', 'He',}
\pysrcoutput{'one', 'I', 'its', 'but', 'or', 'more', ')', 'Mr.', 'up', '(', 'all', 'last',}
\pysrcoutput{'out', 'two', ':', 'other', 'new', 'first', 'year', 'than', 'A', 'about', 'there',}
\pysrcoutput{'when', 'home', 'after', 'In', 'also', 'over', 'It', 'into', 'no', 'But', 'made',}
\pysrcoutput{'her', 'only', 'years', 'time', 'three', 'them', 'some', 'can', 'New', 'him',}
\pysrcoutput{'state', '?', 'any', 'President', 'could', 'before', 'week', 'under', 'against',}
\pysrcoutput{'we', 'now'{]}}}\end{alltt}

{\noindent}
Next, let's inspect the tags that these words have.  First we will do
this in the most obvious (but highly inefficient) way:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }{[}(w,t) \pysrckeyword{for} (w,t) \pysrckeyword{in} nltk.corpus.brown.tagged{\_}words(categories=\pysrcstring{'a'})
\pysrcmore{... }       \pysrckeyword{if} w \pysrckeyword{in} most{\_}freq{\_}words{]}
\pysrcoutput{{[}('The', 'AT'), ('said', 'VBD'), ('an', 'AT'), ('of', 'IN'),}
\pysrcoutput{('`{}`', '`{}`'), ('no', 'AT'), ("'{}'", "'{}'"), ('that', 'CS'),}
\pysrcoutput{('any', 'DTI'), ('.', '.'), ..., ("'{}'", "'{}'"){]}}}\end{alltt}

{\noindent}
A much better approach is to set up a dictionary that maps each of the
100 most frequent words to its most likely tag.  We can do this by
setting up a frequency distribution \texttt{cfd} over the tagged words,
i.e. the frequency of the different tags that occur with each word.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }cfd = nltk.ConditionalFreqDist(nltk.corpus.brown.tagged{\_}words(categories=\pysrcstring{'a'}))}\end{alltt}

{\noindent}
Now for any word that appears in this section of the corpus,
we can determine its most likely tag:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }likely{\_}tags = dict((word, cfd{[}word{]}.max()) \pysrckeyword{for} word \pysrckeyword{in} most{\_}freq{\_}words)
\pysrcprompt{>{}>{}> }likely{\_}tags{[}\pysrcstring{'The'}{]}
\pysrcoutput{'AT'}}\end{alltt}

{\noindent}
Finally, we can create and evaluate a simple tagger that assigns tags to words
based on this table:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }baseline{\_}tagger = nltk.UnigramTagger(model=likely{\_}tags)
\pysrcprompt{>{}>{}> }nltk.tag.accuracy(baseline{\_}tagger, nltk.corpus.brown.tagged{\_}sents(categories=\pysrcstring{'a'}))
\pysrcoutput{0.45578495136941344}}\end{alltt}

This is surprisingly good; just knowing the tags for the 100 most frequent
words enables us to tag nearly half of all words correctly!  Let's see
what it does on some untagged input text:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }baseline{\_}tagger.tag(nltk.corpus.brown.sents(categories=\pysrcstring{'a'}){[}3{]})
\pysrcoutput{{[}('`{}`', '`{}`'), ('Only', None), ('a', 'AT'), ('relative', None),}
\pysrcoutput{('handful', None), ('of', 'IN'), ('such', None), ('reports', None),}
\pysrcoutput{('was', 'BEDZ'), ('received', None), ("'{}'", "'{}'"), (',', ','),}
\pysrcoutput{('the', 'AT'), ('jury', None), ('said', 'VBD'), (',', ','),}
\pysrcoutput{('`{}`', '`{}`'), ('considering', None), ('the', 'AT'), ('widespread', None),}
\pysrcoutput{('interest', None), ('in', 'IN'), ('the', 'AT'), ('election', None),}
\pysrcoutput{(',', ','), ('the', 'AT'), ('number', None), ('of', 'IN'),}
\pysrcoutput{('voters', None), ('and', 'CC'), ('the', 'AT'), ('size', None),}
\pysrcoutput{('of', 'IN'), ('this', 'DT'), ('city', None), ("'{}'", "'{}'"), ('.', '.'){]}}}\end{alltt}

Notice that a lot of these words have been assigned a tag of \texttt{None}.
That is because they were not among the 100 most frequent words.
In these cases we would like to assign the default tag of \texttt{NN},
a process known as backoff.


%___________________________________________________________________________

\subsection{Backoff}
\label{backoff}

How do we combine these taggers?  We want to use the lookup table
first, and if it is unable to assign a tag, then use the default tagger.
We do this by specifying the default tagger as an argument to the
lookup tagger.  The lookup tagger will call the default tagger just
in case it can't assign a tag itself.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }baseline{\_}tagger = nltk.UnigramTagger(model=likely{\_}tags, backoff=nltk.DefaultTagger(\pysrcstring{'NN'}))
\pysrcprompt{>{}>{}> }nltk.tag.accuracy(baseline{\_}tagger, nltk.corpus.brown.tagged{\_}sents(categories=\pysrcstring{'a'}))
\pysrcoutput{0.58177695566561249}}\end{alltt}

We will return to this technique in the context of a broader discussion
on combining taggers in \href{\#sec-combining-taggers}{Section 3.5.6}.


%___________________________________________________________________________

\subsection{Choosing a Good Baseline}
\label{choosing-a-good-baseline}

We can put all this together to write a simple (but somewhat inefficient) program to create and
evaluate lookup taggers having a range of sizes, as shown in \href{\#baseline-tagger}{Listing 3.3}.
We include a backoff tagger that tags everything as a noun.  A consequence
of using this backoff tagger is that the lookup tagger only has to store word/tag
pairs for words other than nouns.
\hypertarget{baseline-tagger}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{performance}(cfd, wordlist):
    lt = dict((word, cfd{[}word{]}.max()) \pysrckeyword{for} word \pysrckeyword{in} wordlist)
    baseline{\_}tagger = nltk.UnigramTagger(model=lt, backoff=nltk.DefaultTagger(\pysrcstring{'NN'}))
    return nltk.tag.accuracy(baseline{\_}tagger, nltk.corpus.brown.tagged{\_}sents(categories=\pysrcstring{'a'}))

\pysrckeyword{def} \pysrcdefname{display}():
    \pysrckeyword{import} pylab
    words{\_}by{\_}freq = nltk.FreqDist(nltk.corpus.brown.words(categories=\pysrcstring{'a'})).sorted()
    cfd = nltk.ConditionalFreqDist(nltk.corpus.brown.tagged{\_}words(categories=\pysrcstring{'a'}))
    sizes = 2 ** pylab.arange(15)
    perfs = {[}performance(cfd, words{\_}by{\_}freq{[}:size{]}) \pysrckeyword{for} size \pysrckeyword{in} sizes{]}
    pylab.plot(sizes, perfs, \pysrcstring{'-bo'})
    pylab.title(\pysrcstring{'Lookup Tagger Performance with Varying Model Size'})
    pylab.xlabel(\pysrcstring{'Model Size'})
    pylab.ylabel(\pysrcstring{'Performance'})
    pylab.show()}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }display()                                  }\end{alltt}
\caption{Lookup Tagger Performance with Varying Model Size}
\vspace{1ex}
\hrule
\end{pylisting}
\hypertarget{tag-lookup}{}\begin{figure}[htbp]
\centering

\scalebox{0.308571}{\includegraphics{../images/tag-lookup.png}}
\caption{Lookup Tagger}\end{figure}

Observe that performance initially increases rapidly as the model size grows, eventually
reaching a plateau, when large increases in model size yield little improvement
in performance.  (This example used the \texttt{pylab} plotting package; we will
return to this later in \href{\#sec-graphics}{Section 5.3.4}).


%___________________________________________________________________________

\subsection{Exercises}
\label{id29}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
◑ Explore the following issues that arise in connection with the lookup tagger:
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
What happens to the tagger performance for the various
model sizes when a backoff tagger is omitted?

\item {} 
Consider the curve in \href{\#tag-lookup}{Figure 3.2}; suggest a
good size for a lookup tagger that balances memory and performance.
Can you come up with scenarios where it would be preferable to
minimize memory usage, or to maximize performance with no regard for memory usage?

\end{list}

\item {} 
◑ What is the upper limit of performance for a lookup tagger,
assuming no limit to the size of its table?  (Hint: write a program
to work out what percentage of tokens of a word are assigned
the most likely tag for that word, on average.)

\end{list}


%___________________________________________________________________________

\section{Getting Better Coverage}
\label{getting-better-coverage}


%___________________________________________________________________________

\subsection{More English Word Classes}
\label{more-english-word-classes}

Two other important word classes are \index{adjectives}\textbf{adjectives} and \index{adverbs}\textbf{adverbs}.
Adjectives describe nouns, and can be used as modifiers
(e.g. \docutilsroleexample{large} in \docutilsroleexample{the large pizza}), or in predicates (e.g. \docutilsroleexample{the
pizza is large}).  English adjectives can be morphologically complex
(e.g.  \docutilsroleexample{fall}$_{V}$\docutilsroleexample{+ing} in \docutilsroleexample{the falling
stocks}).  Adverbs modify verbs to specify the time, manner, place or
direction of the event described by the verb (e.g. \docutilsroleexample{quickly} in
\docutilsroleexample{the stocks fell quickly}).  Adverbs may also modify adjectives
(e.g. \docutilsroleexample{really} in \docutilsroleexample{Mary's teacher was really nice}).

English has several categories of closed class words in addition to
prepositions, such as \index{articles}\textbf{articles} (also often called \index{determiners}\textbf{determiners})
(e.g., \docutilsroleexample{the}, \docutilsroleexample{a}), \index{modals}\textbf{modals} (e.g., \docutilsroleexample{should},
\docutilsroleexample{may}), and \index{personal pronouns}\textbf{personal pronouns} (e.g., \docutilsroleexample{she}, \docutilsroleexample{they}).
Each dictionary and grammar classifies these words differently.

Part-of-speech tags are closely related to the notion of word class
used in syntax.  The assumption in linguistics is that every distinct
word type will be listed in a lexicon (or dictionary), with
information about its pronunciation, syntactic properties and
meaning. A key component of the word's properties will be its
class. When we carry out a syntactic analysis of an example like
\docutilsroleexample{fruit flies like a banana}, we will look up each word in the
lexicon, determine its word class, and then group it into a hierarchy
of phrases, as illustrated in the following parse tree.

\scalebox{0.771429}{\includegraphics{tree_images/book-tree-1.pdf}}

Syntactic analysis will be dealt with in more detail in Part II.
For now, we simply want to make the connection between the labels used in
syntactic parse trees and part-of-speech tags.  \href{\#word-classes}{Table 3.5}
shows the correspondence:
\hypertarget{word-classes}{}
\leavevmode
\begin{longtable}[c]{|p{0.203\locallinewidth}|p{0.121\locallinewidth}|p{0.342\locallinewidth}|}
\hline
\textbf{
Word Class Label
} & \textbf{
Brown Tag
} & \textbf{
Word Class
} \\
\hline
\endhead

Det
 & 
\docutilsrolecategory{AT}
 & 
article
 \\
\hline

N
 & 
\docutilsrolecategory{NN}
 & 
noun
 \\
\hline

V
 & 
\docutilsrolecategory{VB}
 & 
verb
 \\
\hline

Adj
 & 
\docutilsrolecategory{JJ}
 & 
adjective
 \\
\hline

P
 & 
\docutilsrolecategory{IN}
 & 
preposition
 \\
\hline

Card
 & 
\docutilsrolecategory{CD}
 & 
cardinal number
 \\
\hline

-{}-
 & 
\docutilsrolecategory{.}
 & 
sentence-ending punctuation
 \\
\hline
\caption{
Word Class Labels and Brown Corpus Tags
}\end{longtable}


%___________________________________________________________________________

\subsection{Some Diagnostics}
\label{some-diagnostics}

Now that we have examined word classes in detail, we turn to a more
basic question: how do we decide what category a word belongs to in
the first place? In general, linguists use three criteria:
morphological (or formal); syntactic (or distributional); semantic (or
notional).  A \index{morphological}\textbf{morphological} criterion is one that looks at the
internal structure of a word. For example, \docutilsroleexample{-ness} is a suffix
that combines with an adjective to produce a noun. Examples are
\docutilsroleexample{happy} → \docutilsroleexample{happiness}, \docutilsroleexample{ill} → \docutilsroleexample{illness}. So
if we encounter a word that ends in \docutilsroleexample{-ness}, this is very likely
to be a noun.

A \index{syntactic}\textbf{syntactic} criterion refers to the contexts in which a word can
occur. For example, assume that we have already determined the
category of nouns. Then we might say that a syntactic criterion for an
adjective in English is that it can occur immediately before a noun,
or immediately following the words \docutilsroleexample{be} or \docutilsroleexample{very}. According
to these tests, \docutilsroleexample{near} should be categorized as an adjective:
\begin{itemize}
\item[(7)] \begin{itemize}
\item[a.] 
the near window
\end{itemize}
\begin{itemize}
\item[b.] 
The end is (very) near.
\end{itemize}
\end{itemize}

A familiar example of a \index{semantic}\textbf{semantic} criterion is that a noun is
``the name of a person, place or thing''. Within modern linguistics,
semantic criteria for word classes are treated with suspicion, mainly
because they are hard to formalize. Nevertheless, semantic criteria
underpin many of our intuitions about word classes, and enable us to
make a good guess about the categorization of words in languages that
we are unfamiliar with.  For example, if we all we know about the Dutch
\docutilsroleexample{verjaardag} is that it means the same as the English word
\docutilsroleexample{birthday}, then we can guess that \docutilsroleexample{verjaardag} is a noun in
Dutch. However, some care is needed: although we might translate \docutilsroleexample{zij
is vandaag jarig} as \docutilsroleexample{it's her birthday today}, the word
\docutilsroleexample{jarig} is in fact an adjective in Dutch, and has no exact
equivalent in English!

All languages acquire new lexical items. A list of words recently
added to the Oxford Dictionary of English includes \docutilsroleexample{cyberslacker,
fatoush, blamestorm, SARS, cantopop, bupkis, noughties, muggle}, and
\docutilsroleexample{robata}. Notice that all these new words are nouns, and this is
reflected in calling nouns an \titlereference{open class}. By contrast, prepositions
are regarded as a \index{closed class}\textbf{closed class}. That is, there is a limited set of
words belonging to the class (e.g., \docutilsroleexample{above, along, at, below, beside,
between, during, for, from, in, near, on, outside, over, past,
through, towards, under, up, with}), and membership of the set only
changes very gradually over time.


%___________________________________________________________________________

\subsection{Unigram Tagging}
\label{unigram-tagging}

Unigram taggers are based on a simple statistical algorithm:
for each token, assign the tag that is most likely for
that particular token. For example, it will assign the tag \texttt{JJ} to any
occurrence of the word \docutilsroleexample{frequent}, since \docutilsroleexample{frequent} is used as an
adjective (e.g. \docutilsroleexample{a frequent word}) more often than it is used as a
verb (e.g. \docutilsroleexample{I frequent this cafe}).
A unigram tagger behaves just like a lookup tagger (\href{\#sec-lookup-tagger}{Section 3.4.1}),
except there is a more convenient technique for setting it up,
called \index{training}\textbf{training}.  In the following code sample,
we initialize and train a unigram tagger (line \href{\#tag-train}{①}),
use it to tag a sentence, then finally compute the tagger's overall accuracy:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }brown{\_}a = nltk.corpus.brown.tagged{\_}sents(categories=\pysrcstring{'a'})
\pysrcprompt{>{}>{}> }unigram{\_}tagger = nltk.UnigramTagger(brown{\_}a)               ①
\pysrcprompt{>{}>{}> }sent = nltk.corpus.brown.sents(categories=\pysrcstring{'a'}){[}2007{]}
\pysrcprompt{>{}>{}> }unigram{\_}tagger.tag(sent)
\pysrcoutput{{[}('Various', None), ('of', 'IN'), ('the', 'AT'), ('apartments', 'NNS'), ('are', 'BER'),}
\pysrcoutput{('of', 'IN'), ('the', 'AT'), ('terrace', 'NN'), ('type', 'NN'), (',', ','),}
\pysrcoutput{('being', 'BEG'), ('on', 'IN'), ('the', 'AT'), ('ground', 'NN'), ('floor', 'NN'),}
\pysrcoutput{('so', 'QL'), ('that', 'CS'), ('entrance', 'NN'), ('is', 'BEZ'), ('direct', 'JJ'), ('.', '.'){]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.tag.accuracy(unigram{\_}tagger, brown{\_}a)
\pysrcoutput{0.8550331165343994}}\end{alltt}


%___________________________________________________________________________

\subsection{Affix Taggers}
\label{affix-taggers}

Affix taggers are like unigram taggers, except they are trained on
word prefixes or suffixes of a specified length.  (NB. Here we use
\emph{prefix} and \emph{suffix} in the string sense, not the morphological
sense.)  For example, the following tagger will consider suffixes of
length 3 (e.g. \emph{-ize}, \emph{-ion}), for words having at least 5
characters.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }affix{\_}tagger = nltk.AffixTagger(brown{\_}a, affix{\_}length=-2, min{\_}stem{\_}length=3)
\pysrcprompt{>{}>{}> }affix{\_}tagger.tag(sent)
\pysrcoutput{{[}('Various', 'JJ'), ('of', None), ('the', None), ('apartments', 'NNS'), ('are', None),}
\pysrcoutput{('of', None), ('the', None), ('terrace', 'NN'), ('type', None), (',', None),}
\pysrcoutput{('being', 'VBG'), ('on', None), ('the', None), ('ground', 'NN'), ('floor', 'NN'),}
\pysrcoutput{('so', None), ('that', None), ('entrance', 'NN'), ('is', None), ('direct', 'NN'),}
\pysrcoutput{('.', None){]}}}\end{alltt}
\hypertarget{n-gram-taggers}{}

%___________________________________________________________________________

\subsection{N-Gram Taggers}
\label{id30}
\label{n-gram-taggers}

When we perform a language processing task based on unigrams, we are using
one item of context.  In the case of tagging, we only consider the current
token, in isolation from any larger context.  Given such a model, the best
we can do is tag each word with its \emph{a priori} most likely tag.
This means we would tag a word such as \docutilsroleexample{wind} with the same tag,
regardless of whether it appears in the context \docutilsroleexample{the wind} or
\docutilsroleexample{to wind}.

An \index{n-gram tagger}\textbf{n-gram tagger} is a generalization of a unigram tagger whose context is
the current word together with the part-of-speech tags of the
\emph{n-1} preceding tokens, as shown in \href{\#tag-context}{Figure 3.3}. The tag to be
chosen, \emph{t}$_{n}$, is circled, and the context is shaded
in grey. In the example of an n-gram tagger shown in \href{\#tag-context}{Figure 3.3},
we have \emph{n=3}; that is, we consider the tags of the two preceding words in addition
to the current word.  An n-gram tagger
picks the tag that is most likely in the given context.
\hypertarget{tag-context}{}\begin{figure}[htbp]
\centering

\scalebox{0.411429}{\includegraphics{../images/tag-context.png}}
\caption{Tagger Context}\end{figure}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

A 1-gram tagger is another term for a unigram tagger: i.e.,
the context used to tag a token is just the text of the token itself.
2-gram taggers are also called \emph{bigram taggers}, and 3-gram taggers
are called \emph{trigram taggers}.
\end{boxedminipage}
\end{sffamily}
\end{center}

The \texttt{NgramTagger} class uses a tagged training corpus to determine which
part-of-speech tag is most likely for each context.  Here we see
a special case of an n-gram tagger, namely a bigram tagger.
First we train it, then use it to tag untagged sentences:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }bigram{\_}tagger = nltk.BigramTagger(brown{\_}a, cutoff=0)
\pysrcprompt{>{}>{}> }bigram{\_}tagger.tag(sent)
\pysrcoutput{{[}('Various', 'JJ'), ('of', 'IN'), ('the', 'AT'), ('apartments', 'NNS'), ('are', 'BER'),}
\pysrcoutput{('of', 'IN'), ('the', 'AT'), ('terrace', 'NN'), ('type', 'NN'), (',', ','),}
\pysrcoutput{('being', 'BEG'), ('on', 'IN'), ('the', 'AT'), ('ground', 'NN'), ('floor', 'NN'),}
\pysrcoutput{('so', 'CS'), ('that', 'CS'), ('entrance', 'NN'), ('is', 'BEZ'), ('direct', 'JJ'),}
\pysrcoutput{('.', '.'){]}}}\end{alltt}

{\noindent}
As with the other taggers, n-gram taggers assign the tag \docutilsrolecategory{None} to
any token whose context was not seen during training.

As \emph{n} gets larger, the specificity of the contexts increases,
as does the chance that the data we wish to tag contains contexts that
were not present in the training data. This is known as the \emph{sparse
data} problem, and is quite pervasive in NLP. Thus, there is a
trade-off between the accuracy and the coverage of our results (and
this is related to the \index{precision/recall trade-off}\textbf{precision/recall trade-off} in information
retrieval).
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

n-gram taggers should not consider context that crosses a
sentence boundary.  Accordingly, NLTK taggers are designed to work
with lists of sentences, where each sentence is a list of words.  At
the start of a sentence, \emph{t}$_{n-1}$ and preceding
tags are set to \texttt{None}.
\end{boxedminipage}
\end{sffamily}
\end{center}
\hypertarget{sec-combining-taggers}{}

%___________________________________________________________________________

\subsection{Combining Taggers}
\label{combining-taggers}
\label{sec-combining-taggers}

One way to address the trade-off between accuracy and coverage is to
use the more accurate algorithms when we can, but to fall back on
algorithms with wider coverage when necessary. For example, we could
combine the results of a bigram tagger, a unigram tagger, and
a \texttt{regexp{\_}tagger}, as follows:
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Try tagging the token with the bigram tagger.

\item {} 
If the bigram tagger is unable to find a tag for the token, try
the unigram tagger.

\item {} 
If the unigram tagger is also unable to find a tag, use a default tagger.

\end{list}

Most NLTK taggers permit a backoff-tagger to be specified.
The backoff-tagger may itself have a backoff tagger:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }t0 = nltk.DefaultTagger(\pysrcstring{'NN'})
\pysrcprompt{>{}>{}> }t1 = nltk.UnigramTagger(brown{\_}a, backoff=t0)
\pysrcprompt{>{}>{}> }t2 = nltk.BigramTagger(brown{\_}a, backoff=t1)
\pysrcprompt{>{}>{}> }nltk.tag.accuracy(t2, brown{\_}a)
\pysrcoutput{0.88565347972233821}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

We specify the backoff tagger when the tagger is
initialized, so that training can take advantage of the backoff tagger.
Thus, if the bigram tagger would assign
the same tag as its unigram backoff tagger in
a certain context, the bigram tagger discards the training instance.
This keeps the bigram tagger model as small as possible.  We can
further specify that a tagger needs to see more than one instance of a
context in order to retain it, e.g. \texttt{nltk{\linebreak[0]}.{\linebreak[0]}BigramTagger{\linebreak[0]}({\linebreak[0]}sents{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}cutoff{\linebreak[0]}={\linebreak[0]}2{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}backoff{\linebreak[0]}={\linebreak[0]}t1{\linebreak[0]})}
will discard contexts that have only been seen once or twice.
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\subsection{Tagging Unknown Words}
\label{tagging-unknown-words}

Our approach to tagging unknown words still uses backoff to a regular-expression tagger
or a default tagger.  These are unable to make use of context.  Thus, if our tagger
encountered the word \docutilsroleexample{blog}, not seen during training, it would assign it a tag
regardless of whether this word appeared in the context \docutilsroleexample{the blog} or \docutilsroleexample{to blog}.
How can we do better with these unknown words, or \index{out-of-vocabulary}\textbf{out-of-vocabulary} items?

A useful method to tag unknown words based on context is to limit the vocabulary
of a tagger to the most frequent \docutilsrolemath{n} words, and to replace every other word
with a special word \docutilsroleexample{UNK}.  During training, a unigram tagger will probably
learn that this ``word'' is usually a noun.  However, the n-gram taggers will
detect contexts in which it has some other tag.  For example, if the
preceding word is \docutilsroleexample{to} (tagged \texttt{TO}), then \docutilsroleexample{UNK} will probably
be tagged as a verb.  Full exploration of this method is left to the exercises.


%___________________________________________________________________________

\subsection{Storing Taggers}
\label{storing-taggers}

Training a tagger on a large corpus may take several minutes.  Instead of training a tagger
every time we need one, it is convenient to save a trained tagger in a file for later re-use.
Let's save our tagger \texttt{t2} to a file \texttt{t2{\linebreak[0]}.{\linebreak[0]}pkl}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} cPickle \pysrckeyword{import} dump
\pysrcprompt{>{}>{}> }output = open(\pysrcstring{'t2.pkl'}, \pysrcstring{'wb'})
\pysrcprompt{>{}>{}> }dump(t2, output, -1)
\pysrcprompt{>{}>{}> }output.close()}\end{alltt}

{\noindent}
Now, in a separate Python process, we can load our saved tagger.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} cPickle \pysrckeyword{import} load
\pysrcprompt{>{}>{}> }input = open(\pysrcstring{'t2.pkl'}, \pysrcstring{'rb'})
\pysrcprompt{>{}>{}> }tagger = load(input)
\pysrcprompt{>{}>{}> }input.close()}\end{alltt}

{\noindent}
Now let's check that it can be used for tagging.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text = \pysrcstring{"{}"{}"The board's action shows what free enterprise}
\pysrcmore{... }\pysrcstring{    is up against in our complex maze of regulatory laws ."{}"{}"}
\pysrcprompt{>{}>{}> }tokens = text.split()
\pysrcprompt{>{}>{}> }tagger.tag(tokens)
\pysrcoutput{{[}('The', 'AT'), ("board's", 'NN{\$}'), ('action', 'NN'), ('shows', 'NNS'),}
\pysrcoutput{('what', 'WDT'), ('free', 'JJ'), ('enterprise', 'NN'), ('is', 'BEZ'),}
\pysrcoutput{('up', 'RP'), ('against', 'IN'), ('in', 'IN'), ('our', 'PP{\$}'), ('complex', 'JJ'),}
\pysrcoutput{('maze', 'NN'), ('of', 'IN'), ('regulatory', 'NN'), ('laws', 'NNS'), ('.', '.'){]}}}\end{alltt}


%___________________________________________________________________________

\subsection{Exercises}
\label{id31}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼
Train a unigram tagger and run it on some new text.
Observe that some words are not assigned a tag.  Why not?

\item {} 
☼
Train an affix tagger \texttt{AffixTagger{\linebreak[0]}({\linebreak[0]})} and run it on some new text.
Experiment with different settings for the affix length
and the minimum word length.  Can you find a setting that seems
to perform better than the one described above?  Discuss your findings.

\item {} 
☼
Train a bigram tagger with no backoff tagger, and run it on some of the training
data.  Next, run it on some new data.
What happens to the performance of the tagger?  Why?

\item {} 
◑
Write a program that calls \texttt{AffixTagger{\linebreak[0]}({\linebreak[0]})} repeatedly, using
different settings for the affix length and the minimum word length.
What parameter values give the best overall performance?  Why do
you think this is the case?

\item {} 
◑
How serious is the sparse data problem?  Investigate the
performance of n-gram taggers as \docutilsrolemath{n} increases from 1 to 6.
Tabulate the accuracy score.  Estimate the training data required
for these taggers, assuming a vocabulary size of
10$^{5}$ and a tagset size of 10$^{2}$.

\item {} 
◑ Obtain some tagged data for another language, and train and
evaluate a variety of taggers on it.  If the language is
morphologically complex, or if there are any orthographic clues
(e.g. capitalization) to word classes, consider developing a
regular expression tagger for it (ordered after the unigram
tagger, and before the default tagger).  How does the accuracy of
your tagger(s) compare with the same taggers run on English data?
Discuss any issues you encounter in applying these methods to the language.

\item {} 
★
Create a default tagger and various unigram and n-gram taggers,
incorporating backoff, and train them on part of the Brown corpus.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Create three different combinations of the taggers. Test the
accuracy of each combined tagger. Which combination works best?

\item {} 
Try varying the size of the training corpus. How does it affect
your results?

\end{list}

\item {} 
★
Our approach for tagging an unknown word has been to consider the letters of the word
(using \texttt{RegexpTagger{\linebreak[0]}({\linebreak[0]})} and \texttt{AffixTagger{\linebreak[0]}({\linebreak[0]})}), or to ignore the word altogether and tag
it as a noun (using \texttt{nltk{\linebreak[0]}.{\linebreak[0]}DefaultTagger{\linebreak[0]}({\linebreak[0]})}).  These methods will not do well for texts having
new words that are not nouns.
Consider the sentence \docutilsroleexample{I like to blog on Kim's blog}.  If \docutilsroleexample{blog} is a new
word, then looking at the previous tag (\texttt{TO} vs \texttt{NP{\linebreak[0]}{\$}}) would probably be helpful.
I.e. we need a default tagger that is sensitive to the preceding tag.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Create a new kind of unigram tagger that looks at the tag of the previous word,
and ignores the current word.  (The best way to do this is to modify the source
code for \texttt{UnigramTagger{\linebreak[0]}({\linebreak[0]})}, which presumes knowledge of Python classes discussed
in \href{\#oo}{Section 9}.)

\item {} 
Add this tagger to the sequence of backoff taggers (including ordinary trigram
and bigram taggers that look at words), right before the usual default tagger.

\item {} 
Evaluate the contribution of this new unigram tagger.

\end{list}

\item {} 
★
Write code to preprocess tagged training data, replacing all but the most frequent
\docutilsrolemath{n} words with the special word \docutilsroleexample{UNK}.  Train an n-gram backoff tagger
on this data, then use it to tag some new text.  Note that you will have
to preprocess the text to replace unknown words with \docutilsroleexample{UNK}, and post-process
the tagged output to replace the \docutilsroleexample{UNK} words with the words from the original
input.

\end{list}


%___________________________________________________________________________

\section{Summary}
\label{id32}
\begin{itemize}
\item {} 
Words can be grouped into classes, such as nouns, verbs, adjectives, and adverbs.
These classes are known as lexical categories or parts of speech.
Parts of speech are assigned short labels, or tags, such as \texttt{NN}, \texttt{VB},

\item {} 
The process of automatically assigning parts of speech to words in text
is called part-of-speech tagging, POS tagging, or just tagging.

\item {} 
Some linguistic corpora, such as the Brown Corpus, have been POS tagged.

\item {} 
A variety of tagging methods are possible, e.g.
default tagger, regular expression tagger, unigram tagger and n-gram taggers.
These can be combined using a technique known as backoff.

\item {} 
Taggers can be trained and evaluated using tagged corpora.

\item {} 
Part-of-speech tagging is an important, early example of a sequence
classification task in NLP: a classification decision at any one point
in the sequence makes use of words and tags in the local context.

\end{itemize}
\hypertarget{sec-tag-further-reading}{}

%___________________________________________________________________________

\section{Further Reading}
\label{id33}
\label{sec-tag-further-reading}

For more examples of tagging with NLTK, please see the
guide at \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}tag{\linebreak[0]}.{\linebreak[0]}html}.
Chapters 4 and 5 of \cite{jurafskymartin2008} contain more advanced
material on n-grams and part-of-speech tagging.

There are several other important approaches to tagging involving
\emph{Transformation-Based Learning}, \emph{Markov Modeling}, and \emph{Finite State
Methods}.  (We will discuss some of these in \href{\#chap-data-intensive}{Chapter 4}.)
In \href{\#chap-chunk}{Chapter 6} we will see a generalization of tagging called \emph{chunking} in which a
contiguous sequence of words is assigned a single tag.

Part-of-speech tagging is just one kind of tagging, one that
does not depend on deep linguistic analysis.
There are many other kinds of tagging.  Words
can be tagged with directives to a speech synthesizer, indicating
which words should be emphasized.  Words can be tagged with sense
numbers, indicating which sense of the word was used.  Words can also
be tagged with morphological features.
Examples of each of these kinds of tags are shown below.
For space reasons, we only show the tag for a single
word. Note also that the first two examples use XML-style
tags, where elements in angle brackets enclose the word that is
tagged.
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
\emph{Speech Synthesis Markup Language (W3C SSML):}
\texttt{That{\linebreak[0]} \pysrckeyword{is} {\linebreak[0]}a{\linebreak[0]} {\linebreak[0]}<{\linebreak[0]}emphasis{\linebreak[0]}>{\linebreak[0]}big{\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}emphasis{\linebreak[0]}>{\linebreak[0]} {\linebreak[0]}car{\linebreak[0]}!}

\item {} 
\emph{SemCor: Brown Corpus tagged with WordNet senses:}
\texttt{Space{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}any{\linebreak[0]} {\linebreak[0]}<{\linebreak[0]}wf{\linebreak[0]} {\linebreak[0]}pos{\linebreak[0]}=\pysrcstring{"{\linebreak[0]}NN{\linebreak[0]}"} {\linebreak[0]}lemma{\linebreak[0]}=\pysrcstring{"{\linebreak[0]}form{\linebreak[0]}"} {\linebreak[0]}wnsn{\linebreak[0]}=\pysrcstring{"{\linebreak[0]}4{\linebreak[0]}"}>{\linebreak[0]}form{\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}wf{\linebreak[0]}>
\pysrckeyword{is} {\linebreak[0]}completely{\linebreak[0]} {\linebreak[0]}measured{\linebreak[0]} {\linebreak[0]}by{\linebreak[0]} {\linebreak[0]}the{\linebreak[0]} {\linebreak[0]}three{\linebreak[0]} {\linebreak[0]}dimensions{\linebreak[0]}.}
(Wordnet form/nn sense 4: ``shape, form, configuration,
contour, conformation'')

\item {} 
\emph{Morphological tagging, from the Turin University Italian Treebank:}
\texttt{E\pysrcstring{'{\linebreak[0]} {\linebreak[0]}italiano{\linebreak[0]} {\linebreak[0]},{\linebreak[0]} {\linebreak[0]}come{\linebreak[0]} {\linebreak[0]}progetto{\linebreak[0]} {\linebreak[0]}e{\linebreak[0]} {\linebreak[0]}realizzazione{\linebreak[0]} {\linebreak[0]},{\linebreak[0]} {\linebreak[0]}il}
\pysrcstring{primo{\linebreak[0]} {\linebreak[0]}({\linebreak[0]}PRIMO{\linebreak[0]} {\linebreak[0]}ADJ{\linebreak[0]} {\linebreak[0]}ORDIN{\linebreak[0]} {\linebreak[0]}M{\linebreak[0]} {\linebreak[0]}SING{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}porto{\linebreak[0]} {\linebreak[0]}turistico{\linebreak[0]} {\linebreak[0]}dell{\linebreak[0]}'} {\linebreak[0]}Albania{\linebreak[0]} {\linebreak[0]}.}

\end{list}

Tagging exhibits several properties that are characteristic of natural
language processing.  First, tagging involves \emph{classification}: words have
properties; many words share the same property (e.g. \texttt{cat} and \texttt{dog}
are both nouns), while some words can have multiple such properties
(e.g. \texttt{wind} is a noun and a verb).  Second, in tagging, disambiguation
occurs via \emph{representation}: we augment the representation of tokens with
part-of-speech tags.  Third, training a tagger involves \emph{sequence learning
from annotated corpora}.  Finally, tagging uses \emph{simple, general, methods}
such as conditional frequency distributions and transformation-based learning.

Note that tagging is also performed at higher levels.  Here is an example
of dialogue act tagging, from the NPS Chat Corpus \cite{forsyth2007}, included with
NLTK.

\begin{lineblock}{0em}
\item[] Statement  User117 Dude..., I wanted some of that
\item[] ynQuestion User120 m I missing something?
\item[] Bye        User117 I'm gonna go fix food, I'll be back later.
\item[] System     User122 JOIN
\item[] System     User2   slaps User122 around a bit with a large trout.
\item[] Statement  User121 18/m pm me if u tryin to chat
\end{lineblock}

List of available taggers:
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}www{\linebreak[0]}-{\linebreak[0]}nlp{\linebreak[0]}.{\linebreak[0]}stanford{\linebreak[0]}.{\linebreak[0]}edu{\linebreak[0]}/{\linebreak[0]}links{\linebreak[0]}/{\linebreak[0]}statnlp{\linebreak[0]}.{\linebreak[0]}html}


%___________________________________________________________________________

\section{Appendix: Brown Tag Set}
\label{appendix-brown-tag-set}
\hypertarget{appendix}{}
\href{\#brown-tags}{Table 3.6} gives a sample of closed class words, following
the classification of the Brown Corpus. (Note that part-of-speech tags
may be presented as either upper-case or lower-case strings — the
case difference is not significant.)
\hypertarget{brown-tags}{}
\leavevmode
\begin{longtable}[c]{|p{0.028\locallinewidth}|p{0.199\locallinewidth}|p{0.718\locallinewidth}|}
\hline

AP
 & 
determiner/pronoun, post-determiner
 & 
many other next more last former little several enough most least only very few fewer past same
 \\
\hline

AT
 & 
article
 & 
the an no a every th' ever' ye
 \\
\hline

CC
 & 
conjunction, coordinating
 & 
and or but plus {\&} either neither nor yet 'n' and/or minus an'
 \\
\hline

CS
 & 
conjunction, subordinating
 & 
that as after whether before while like because if since for than until so unless though providing once lest till whereas whereupon supposing albeit then
 \\
\hline

IN
 & 
preposition
 & 
of in for by considering to on among at through with under into regarding than since despite ...
 \\
\hline

MD
 & 
modal auxiliary
 & 
should may might will would must can could shall ought need wilt
 \\
\hline

PN
 & 
pronoun, nominal
 & 
none something everything one anyone nothing nobody everybody everyone anybody anything someone no-one nothin'
 \\
\hline

PPL
 & 
pronoun, singular, reflexive
 & 
itself himself myself yourself herself oneself ownself
 \\
\hline

PP{\$}
 & 
determiner, possessive
 & 
our its his their my your her out thy mine thine
 \\
\hline

PP{\$}{\$}
 & 
pronoun, possessive
 & 
ours mine his hers theirs yours
 \\
\hline

PPS
 & 
pronoun, personal, nom, 3rd pers sng
 & 
it he she thee
 \\
\hline

PPSS
 & 
pronoun, personal, nom, not 3rd pers sng
 & 
they we I you ye thou you'uns
 \\
\hline

WDT
 & 
WH-determiner
 & 
which what whatever whichever
 \\
\hline

WPS
 & 
WH-pronoun, nominative
 & 
that who whoever whosoever what whatsoever
 \\
\hline
\caption{
Some English Closed Class Words, with Brown Tag
}\end{longtable}


%___________________________________________________________________________

\subsection{Acknowledgments}
\label{id36}
% Dutch example: http://www.askoxford.com/pressroom/archive/odelaunch/ 
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
\hypertarget{chap-data-intensive}{}\setcounter{chapter}{3}

%___________________________________________________________________________

\chapter{Data-Intensive Language Processing}
\label{data-intensive-language-processing}
\label{chap-data-intensive}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

this chapter is currently in preparation
\end{boxedminipage}
\end{sffamily}
\end{center}
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}

\part{PARSING}

\textbf{Introduction to Part II}

Part II covers the linguistic and computational analysis of sentences.
We will see that sentences have systematic structure;
we use this to communicate who did what to whom.
Linguistic structures are formalized using context-free grammars,
and processed computationally using parsers.
Various extensions are covered, including chart parsers and probabilistic parsers.
Part II also introduces the techniques in structured programming
needed for implementing grammars and parsers.
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
\hypertarget{chap-structured-programming}{}\setcounter{chapter}{4}

%___________________________________________________________________________

\chapter{Structured Programming in Python}
\label{structured-programming-in-python}
\label{chap-structured-programming}
% TODO: introduce the concept of search, backtracking 
% TODO: introduce diagnostic print statements 
% TODO: timeit() 
% TODO: cover yield statement, as promised in words chapter 
% TODO: exercise based on word squares http://itre.cis.upenn.edu/~myl/languagelog/archives/002679.html 
% TODO: docstrings, commenting practice 


%___________________________________________________________________________

\section{Introduction}
\label{id37}

In Part I you had an intensive introduction to Python (\href{\#chap-programming}{Chapter 1})
followed by chapters on words, tags, and chunks (Chapters \href{\#chap-words}{2}-\href{\#chap-chunk}{6}).
These chapters contain many examples and exercises
that should have helped you consolidate your Python skills
and apply them to simple NLP tasks.  So far our programs — and the data we have
been processing — have been relatively unstructured.
In Part II we will focus on \docutilsroleemphasis{structure}: i.e. structured programming
with structured data.

In this chapter we will review key programming concepts and explain
many of the minor points that could easily trip you up.  More fundamentally,
we will introduce important concepts in structured programming that
help you write readable, well-organized programs that you and others will
be able to re-use.  Each section is independent, so you can easily select
what you most need to learn and concentrate on that.
As before, this chapter contains many examples and
exercises (and as before, some exercises introduce new material).
Readers new to programming should work through them carefully
and consult other introductions to programming if necessary;
experienced programmers can quickly skim this chapter.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Remember that our program samples assume you
begin your interactive session or your program with: \texttt{\pysrckeyword{import} {\linebreak[0]}nltk{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}re{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}pprint}
\end{boxedminipage}
\end{sffamily}
\end{center}
\hypertarget{sec-back-to-the-basics}{}

%___________________________________________________________________________

\section{Back to the Basics}
\label{back-to-the-basics}
\label{sec-back-to-the-basics}

Let's begin by revisiting some of the fundamental operations and data structures
required for natural language processing in Python.  It is important to appreciate
several finer points in order to write Python programs that are not
only correct but also \docutilsroleemphasis{idiomatic} — by this, we mean using the
features of the Python language in a natural and concise way. To
illustrate, here is a technique for iterating over the members
of a list by initializing an index \texttt{i} and then incrementing the
index each time we pass through the loop:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = {[}\pysrcstring{'I'}, \pysrcstring{'am'}, \pysrcstring{'the'}, \pysrcstring{'Walrus'}{]}
\pysrcprompt{>{}>{}> }i = 0
\pysrcprompt{>{}>{}> }while i < len(sent):
\pysrcmore{... }    \pysrckeyword{print} sent{[}i{]}.lower(),
\pysrcmore{... }    i += 1
\pysrcoutput{i am the walrus}}\end{alltt}

Although this does the job, it is not idiomatic Python. By
contrast, Python's \texttt{\pysrckeyword{for}} statement allows us to achieve the same
effect much more succinctly:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = {[}\pysrcstring{'I'}, \pysrcstring{'am'}, \pysrcstring{'the'}, \pysrcstring{'Walrus'}{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} s \pysrckeyword{in} sent:
\pysrcmore{... }    \pysrckeyword{print} s.lower(),
\pysrcoutput{i am the walrus}}\end{alltt}

We'll start with the most innocuous operation of all: \docutilsroleemphasis{assignment}.
Then we will look at sequence types in detail.
\hypertarget{sec-assignment}{}

%___________________________________________________________________________

\subsection{Assignment}
\label{assignment}
\label{sec-assignment}

Python's assignment statement operates on \docutilsroleemphasis{values}.  But what is a value?
Consider the following code fragment:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }word1 = \pysrcstring{'Monty'}
\pysrcprompt{>{}>{}> }word2 = word1              ①
\pysrcprompt{>{}>{}> }word1 = \pysrcstring{'Python'}           ②
\pysrcprompt{>{}>{}> }word2
\pysrcoutput{'Monty'}}\end{alltt}

This code shows that when we write \texttt{word2{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}word1} in line \href{\#assignment1}{①},
the value of \texttt{word1} (the string \texttt{\pysrcstring{'{\linebreak[0]}Monty{\linebreak[0]}'}}) is assigned to \texttt{word2}.
That is, \texttt{word2} is a \index{copy}\textbf{copy} of \texttt{word1}, so when we overwrite
\texttt{word1} with a new string \texttt{\pysrcstring{'{\linebreak[0]}Python{\linebreak[0]}'}} in line \href{\#assignment2}{②}, the value
of \texttt{word2} is not affected.

However, assignment statements do not always involve making copies in this way.
An important subtlety of Python is that the ``value'' of a structured object
(such as a list) is actually a \docutilsroleemphasis{reference} to the object.  In the
following example, line \href{\#assignment3}{①}
assigns the reference of \texttt{list1} to the new variable \texttt{list2}.
When we modify something inside \texttt{list1} on line \href{\#assignment4}{②}, we can see
that the contents of \texttt{list2} have also been changed.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }list1 = {[}\pysrcstring{'Monty'}, \pysrcstring{'Python'}{]}
\pysrcprompt{>{}>{}> }list2 = list1             ①
\pysrcprompt{>{}>{}> }list1{[}1{]} = \pysrcstring{'Bodkin'}       ②
\pysrcprompt{>{}>{}> }list2
\pysrcoutput{{[}'Monty', 'Bodkin'{]}}}\end{alltt}
\hypertarget{array-memory}{}\begin{figure}[htbp]
\centering

\scalebox{0.102857}{\includegraphics{../images/array-memory.png}}
\caption{List Assignment and Computer Memory}\end{figure}

{\noindent}
Thus line \href{\#assignment3}{①} does not copy the contents of the
variable, only its ``object reference''.
To understand what is going on here, we need to
know how lists are stored in the computer's memory.
In \href{\#array-memory}{Figure 5.1}, we see that a list \texttt{sent1} is
a reference to an object stored at location 3133 (which is
itself a series of pointers to other locations holding strings).
When we assign \texttt{sent2{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}sent1}, it is just the object reference
3133 that gets copied.


%___________________________________________________________________________

\subsection{Sequences: Strings, Lists and Tuples}
\label{sequences-strings-lists-and-tuples}

We have seen three kinds of sequence object: strings, lists, and tuples.
As sequences, they have some common properties: they can be indexed and they have
a length:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text = \pysrcstring{'I turned off the spectroroute'}
\pysrcprompt{>{}>{}> }words = {[}\pysrcstring{'I'}, \pysrcstring{'turned'}, \pysrcstring{'off'}, \pysrcstring{'the'}, \pysrcstring{'spectroroute'}{]}
\pysrcprompt{>{}>{}> }pair = (6, \pysrcstring{'turned'})
\pysrcprompt{>{}>{}> }text{[}2{]}, words{[}3{]}, pair{[}1{]}
\pysrcoutput{('t', 'the', 'turned')}
\pysrcoutput{}\pysrcprompt{>{}>{}> }len(text), len(words), len(pair)
\pysrcoutput{(29, 5, 2)}}\end{alltt}

{\noindent}
We can iterate over the items in a sequence \texttt{s} in a variety of useful ways,
as shown in \href{\#python-sequence}{Table 5.1}.
\hypertarget{python-sequence}{}
\leavevmode
\begin{longtable}[c]{|p{0.455\locallinewidth}|p{0.485\locallinewidth}|}
\hline
\textbf{
Python Expression
} & \textbf{
Comment
} \\
\hline
\endhead

\texttt{\pysrckeyword{for} {\linebreak[0]}item{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}s}
 & 
iterate over the items of \texttt{s}
 \\
\hline

\texttt{\pysrckeyword{for} {\linebreak[0]}item{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}sorted{\linebreak[0]}({\linebreak[0]}s{\linebreak[0]})}
 & 
iterate over the items of \texttt{s} in order
 \\
\hline

\texttt{\pysrckeyword{for} {\linebreak[0]}item{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}set{\linebreak[0]}({\linebreak[0]}s{\linebreak[0]})}
 & 
iterate over unique elements of \texttt{s}
 \\
\hline

\texttt{\pysrckeyword{for} {\linebreak[0]}item{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}reversed{\linebreak[0]}({\linebreak[0]}s{\linebreak[0]})}
 & 
iterate over elements of \texttt{s} in reverse
 \\
\hline

\texttt{\pysrckeyword{for} {\linebreak[0]}item{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}set{\linebreak[0]}({\linebreak[0]}s{\linebreak[0]}){\linebreak[0]}.{\linebreak[0]}difference{\linebreak[0]}({\linebreak[0]}t{\linebreak[0]})}
 & 
iterate over elements of \texttt{s} not in \texttt{t}
 \\
\hline

\texttt{\pysrckeyword{for} {\linebreak[0]}item{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}random{\linebreak[0]}.{\linebreak[0]}shuffle{\linebreak[0]}({\linebreak[0]}s{\linebreak[0]})}
 & 
iterate over elements of \texttt{s} in random order
 \\
\hline
\caption{
Various ways to iterate over sequences
}\end{longtable}

The sequence functions illustrated in \href{\#python-sequence}{Table 5.1} can be combined
in various ways; for example, to get unique elements of \texttt{s} sorted
in reverse, use \texttt{reversed{\linebreak[0]}({\linebreak[0]}sorted{\linebreak[0]}({\linebreak[0]}set{\linebreak[0]}({\linebreak[0]}s{\linebreak[0]}){\linebreak[0]}){\linebreak[0]})}.

We can convert between these sequence types.  For example,
\texttt{tuple{\linebreak[0]}({\linebreak[0]}s{\linebreak[0]})} converts any kind of sequence into a tuple, and
\texttt{list{\linebreak[0]}({\linebreak[0]}s{\linebreak[0]})} converts any kind of sequence into a list.
We can convert a list of strings to a single string using the
\texttt{join{\linebreak[0]}({\linebreak[0]})} function, e.g. \texttt{\pysrcstring{'{\linebreak[0]}:{\linebreak[0]}'}.{\linebreak[0]}join{\linebreak[0]}({\linebreak[0]}words{\linebreak[0]})}.

Notice in the above code sample that we computed multiple values on a
single line, separated by commas.  These comma-separated expressions
are actually just tuples — Python allows us to omit the
parentheses around tuples if there is no ambiguity. When we print a
tuple, the parentheses are always displayed. By using tuples in this
way, we are implicitly aggregating items together.

In the next example, we use tuples to re-arrange the
contents of our list.  (We can omit the parentheses
because the comma has higher precedence than assignment.)
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }words{[}2{]}, words{[}3{]}, words{[}4{]} = words{[}3{]}, words{[}4{]}, words{[}2{]}
\pysrcprompt{>{}>{}> }words
\pysrcoutput{{[}'I', 'turned', 'the', 'spectroroute', 'off'{]}}}\end{alltt}

{\noindent}
This is an idiomatic and readable way to move items inside a list.
It is equivalent to the following traditional way of doing such
tasks that does not use tuples (notice that this method needs a
temporary variable \texttt{tmp}).
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tmp = words{[}2{]}
\pysrcprompt{>{}>{}> }words{[}2{]} = words{[}3{]}
\pysrcprompt{>{}>{}> }words{[}3{]} = words{[}4{]}
\pysrcprompt{>{}>{}> }words{[}4{]} = tmp}\end{alltt}

As we have seen, Python has sequence functions such as \texttt{sorted{\linebreak[0]}({\linebreak[0]})} and \texttt{reversed{\linebreak[0]}({\linebreak[0]})}
that rearrange the items of a sequence.  There are also functions that
modify the \docutilsroleemphasis{structure} of a sequence and which can be handy for
language processing.  Thus, \texttt{zip{\linebreak[0]}({\linebreak[0]})} takes
the items of two sequences and ``zips'' them together into a single list of pairs.
Given a sequence \texttt{s}, \texttt{enumerate{\linebreak[0]}({\linebreak[0]}s{\linebreak[0]})} returns an iterator that
produces a pair of an index and the item at that index.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }words = {[}\pysrcstring{'I'}, \pysrcstring{'turned'}, \pysrcstring{'off'}, \pysrcstring{'the'}, \pysrcstring{'spectroroute'}{]}
\pysrcprompt{>{}>{}> }tags = {[}\pysrcstring{'NNP'}, \pysrcstring{'VBD'}, \pysrcstring{'IN'}, \pysrcstring{'DT'}, \pysrcstring{'NN'}{]}
\pysrcprompt{>{}>{}> }zip(words, tags)
\pysrcoutput{{[}('I', 'NNP'), ('turned', 'VBD'), ('off', 'IN'),}
\pysrcoutput{('the', 'DT'), ('spectroroute', 'NN'){]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }list(enumerate(words))
\pysrcoutput{{[}(0, 'I'), (1, 'turned'), (2, 'off'), (3, 'the'), (4, 'spectroroute'){]}}}\end{alltt}


%___________________________________________________________________________

\subsection{Combining Different Sequence Types}
\label{combining-different-sequence-types}

Let's combine our knowledge of these three sequence types, together with list
comprehensions, to perform the task of sorting the words in a string by
their length.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }words = \pysrcstring{'I turned off the spectroroute'}.split()     ①
\pysrcprompt{>{}>{}> }wordlens = {[}(len(word), word) \pysrckeyword{for} word \pysrckeyword{in} words{]}    ②
\pysrcprompt{>{}>{}> }wordlens
\pysrcoutput{{[}(1, 'I'), (6, 'turned'), (3, 'off'), (3, 'the'), (12, 'spectroroute'){]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }wordlens.sort()                                     ③
\pysrcprompt{>{}>{}> }\pysrcstring{' '}.join({[}word \pysrckeyword{for} (count, word) \pysrckeyword{in} wordlens{]})      ④
\pysrcoutput{'I off the turned spectroroute'}}\end{alltt}

Each of the above lines of code contains a significant feature.
Line \href{\#sequence1}{①} demonstrates that a simple string is actually
an object with methods defined on it, such as \texttt{split{\linebreak[0]}({\linebreak[0]})}.
Line \href{\#sequence2}{②} shows the construction of a list of tuples,
where each tuple consists of a number (the word length) and the
word, e.g. \texttt{({\linebreak[0]}3{\linebreak[0]},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}the{\linebreak[0]}'})}.  Line \href{\#sequence3}{③} sorts the list,
modifying the list in-place.  Finally, line \href{\#sequence4}{④} discards
the length information then joins the words back into a single string.

We began by talking about the commonalities in these sequence types,
but the above code illustrates important differences in their
roles.  First, strings appear at the beginning and the end: this is
typical in the context where our program is reading in some text and
producing output for us to read.  Lists and tuples are used in the
middle, but for different purposes.  A list is typically a sequence of
objects all having the \docutilsroleemphasis{same type}, of \docutilsroleemphasis{arbitrary length}.  We often
use lists to hold sequences of words.  In contrast,
a tuple is typically a collection of objects of \docutilsroleemphasis{different types}, of
\docutilsroleemphasis{fixed length}.  We often use a tuple to hold a \index{record}\textbf{record},
a collection of different \index{fields}\textbf{fields} relating to some entity.
This distinction between the use of lists and tuples takes some
getting used to,
so here is another example:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }lexicon = {[}
\pysrcmore{... }    (\pysrcstring{'the'}, \pysrcstring{'DT'}, {[}\pysrcstring{'Di:'}, \pysrcstring{'D@'}{]}),
\pysrcmore{... }    (\pysrcstring{'off'}, \pysrcstring{'IN'}, {[}\pysrcstring{'Qf'}, \pysrcstring{'O:f'}{]})
\pysrcmore{... }{]}}\end{alltt}

{\noindent} Here, a lexicon is represented as a list because it is a
collection of objects of a single type — lexical entries —
of no predetermined length.  An individual entry is represented as a
tuple because it is a collection of objects with different
interpretations, such as the orthographic form, the part of speech,
and the pronunciations represented in the
\href{http://www.phon.ucl.ac.uk/home/sampa/index.html}{SAMPA} computer
readable phonetic alphabet.  Note that these pronunciations are stored
using a list. (Why?)

The distinction between lists and tuples has been described in terms of
usage.  However, there is a more fundamental difference: in Python,
lists are \index{mutable}\textbf{mutable}, while tuples are \index{immutable}\textbf{immutable}.  In other
words, lists can be modified, while tuples cannot.  Here are some of
the operations on lists that do in-place modification of the list.
None of these operations is permitted on a tuple, a fact you should
confirm for yourself.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }lexicon.sort()
\pysrcprompt{>{}>{}> }lexicon{[}1{]} = (\pysrcstring{'turned'}, \pysrcstring{'VBD'}, {[}\pysrcstring{'t3:nd'}, \pysrcstring{'t3`nd'}{]})
\pysrcprompt{>{}>{}> }\pysrckeyword{del} lexicon{[}0{]}}\end{alltt}
\hypertarget{stacks-and-queues}{}

%___________________________________________________________________________

\subsection{Stacks and Queues}
\label{id38}
\label{stacks-and-queues}

Lists are a particularly versatile data type.  We can use lists to
implement higher-level data types such as stacks and queues.
A \index{stack}\textbf{stack} is a container that has a \index{last-in-first-out}\textbf{last-in-first-out} policy
for adding and removing items (see \href{\#stack-queue}{Figure 5.2}).
\hypertarget{stack-queue}{}\begin{figure}[htbp]
\centering

\scalebox{0.154286}{\includegraphics{../images/stack-queue.png}}
\caption{Stacks and Queues}\end{figure}

Stacks are used to keep track of the current context in
computer processing of natural languages (and programming languages too).
We will seldom have to deal with stacks explicitly, as the implementation
of NLTK parsers, treebank corpus readers, (and even Python functions),
all use stacks behind the scenes.
However, it is important to understand what stacks are and how they work.
\hypertarget{check-parens}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{check{\_}parens}(tokens):
    stack = {[}{]}
    \pysrckeyword{for} token \pysrckeyword{in} tokens:
        \pysrckeyword{if} token == \pysrcstring{'('}:     \pysrccomment{{\#} push}
            stack.append(token)
        \pysrckeyword{elif} token == \pysrcstring{')'}:   \pysrccomment{{\#} pop}
            stack.pop()
    return stack}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }phrase = \pysrcstring{"( the cat ) ( sat ( on ( the mat )"}
\pysrcprompt{>{}>{}> }\pysrckeyword{print} check{\_}parens(phrase.split())
\pysrcoutput{{[}'(', '('{]}}}\end{alltt}
\caption{Check parentheses are balanced}
\vspace{1ex}
\hrule
\end{pylisting}

In Python, we can treat a list as a stack by limiting ourselves to the three
operations defined on stacks: \texttt{append{\linebreak[0]}({\linebreak[0]}item{\linebreak[0]})} (to push \texttt{item} onto the stack),
\texttt{\pysrcbuiltin{pop}({\linebreak[0]})} to pop the item off the top of the stack, and \texttt{{[}{\linebreak[0]}-{\linebreak[0]}1{\linebreak[0]}{]}} to access the
item on the top of the stack.  \href{\#check-parens}{Listing 5.1} processes a sentence with
phrase markers, and checks that the parentheses are balanced.
The loop pushes material onto the stack when it gets an open parenthesis,
and pops the stack when it gets a close parenthesis.
We see that two are left on the stack at the end;
i.e. the parentheses are not balanced.

Although \href{\#check-parens}{Listing 5.1} is a useful illustration of stacks,
it is overkill because we could have done a direct count:
\texttt{phrase{\linebreak[0]}.{\linebreak[0]}count{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}({\linebreak[0]}'}){\linebreak[0]} {\linebreak[0]}={\linebreak[0]}={\linebreak[0]} {\linebreak[0]}phrase{\linebreak[0]}.{\linebreak[0]}count{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}){\linebreak[0]}'})}.  However, we
can use stacks for more sophisticated processing of strings
containing nested structure, as shown in \href{\#convert-parens}{Listing 5.2}.
Here we build a (potentially deeply-nested) list of lists.
Whenever a token other than a parenthesis is encountered,
we add it to a list at the appropriate level of nesting.
The stack cleverly keeps track of this level of nesting, exploiting
the fact that the item at the top of the stack is actually shared with a
more deeply nested item.  (Hint: add diagnostic print statements to
the function to help you see what it is doing.)
\hypertarget{convert-parens}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{convert{\_}parens}(tokens):
    stack = {[}{[}{]}{]}
    \pysrckeyword{for} token \pysrckeyword{in} tokens:
        \pysrckeyword{if} token == \pysrcstring{'('}:     \pysrccomment{{\#} push}
            sublist = {[}{]}
            stack{[}-1{]}.append(sublist)
            stack.append(sublist)
        \pysrckeyword{elif} token == \pysrcstring{')'}:   \pysrccomment{{\#} pop}
            stack.pop()
        \pysrckeyword{else}:                \pysrccomment{{\#} update top of stack}
            stack{[}-1{]}.append(token)
    return stack{[}0{]}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }phrase = \pysrcstring{"( the cat ) ( sat ( on ( the mat ) ) )"}
\pysrcprompt{>{}>{}> }\pysrckeyword{print} convert{\_}parens(phrase.split())
\pysrcoutput{{[}{[}'the', 'cat'{]}, {[}'sat', {[}'on', {[}'the', 'mat'{]}{]}{]}{]}}}\end{alltt}
\caption{Convert a nested phrase into a nested list using a stack}
\vspace{1ex}
\hrule
\end{pylisting}

Lists can be used to represent another important data structure.
A \index{queue}\textbf{queue} is a container that has a \index{first-in-first-out}\textbf{first-in-first-out} policy
for adding and removing items (see \href{\#stack-queue}{Figure 5.2}).
Queues are used for scheduling activities or resources.
As with stacks, we will seldom have to deal with queues explicitly,
as the implementation of NLTK n-gram taggers (\href{\#n-gram-taggers}{Section 3.5.5})
and chart parsers (\href{\#chart-parsing}{Section 8.2}) use queues behind the scenes.
However, we will take a brief look at how queues are implemented using lists.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }queue = {[}\pysrcstring{'the'}, \pysrcstring{'cat'}, \pysrcstring{'sat'}{]}
\pysrcprompt{>{}>{}> }queue.append(\pysrcstring{'on'})
\pysrcprompt{>{}>{}> }queue.append(\pysrcstring{'the'})
\pysrcprompt{>{}>{}> }queue.append(\pysrcstring{'mat'})
\pysrcprompt{>{}>{}> }queue.pop(0)
\pysrcoutput{'the'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }queue.pop(0)
\pysrcoutput{'cat'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }queue
\pysrcoutput{{[}'sat', 'on', 'the', 'mat'{]}}}\end{alltt}
\hypertarget{sec-more-list-comprehensions}{}

%___________________________________________________________________________

\subsection{More List Comprehensions}
\label{more-list-comprehensions}
\label{sec-more-list-comprehensions}

You may recall that in \href{\#chap-words}{Chapter 2}, we introduced list
comprehensions, with examples like the following:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = {[}\pysrcstring{'The'}, \pysrcstring{'dog'}, \pysrcstring{'gave'}, \pysrcstring{'John'}, \pysrcstring{'the'}, \pysrcstring{'newspaper'}{]}
\pysrcprompt{>{}>{}> }{[}word.lower() \pysrckeyword{for} word \pysrckeyword{in} sent{]}
\pysrcoutput{{[}'the', 'dog', 'gave', 'john', 'the', 'newspaper'{]}}}\end{alltt}

List comprehensions are a convenient and readable way to express
list operations in Python, and they have a wide range of uses in natural language
processing.  In this section we will see some more examples.
The first of these takes successive overlapping slices of size
\texttt{n} (a \index{sliding window}\textbf{sliding window}) from a list
(pay particular attention to the range of the variable \texttt{i}).
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = {[}\pysrcstring{'The'}, \pysrcstring{'dog'}, \pysrcstring{'gave'}, \pysrcstring{'John'}, \pysrcstring{'the'}, \pysrcstring{'newspaper'}{]}
\pysrcprompt{>{}>{}> }n = 3
\pysrcprompt{>{}>{}> }{[}sent{[}i:i+n{]} \pysrckeyword{for} i \pysrckeyword{in} range(len(sent)-n+1){]}
\pysrcoutput{{[}{[}'The', 'dog', 'gave'{]},}
\pysrcoutput{ {[}'dog', 'gave', 'John'{]},}
\pysrcoutput{ {[}'gave', 'John', 'the'{]},}
\pysrcoutput{ {[}'John', 'the', 'newspaper'{]}{]}}}\end{alltt}

You can also use list comprehensions for a kind of multiplication (or \index{cartesian product}\textbf{cartesian product}).
Here we generate all combinations of two determiners, two adjectives, and two nouns.
The list comprehension is split across three lines for readability.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }{[}(dt,jj,nn) \pysrckeyword{for} dt \pysrckeyword{in} (\pysrcstring{'two'}, \pysrcstring{'three'})
\pysrcmore{... }            \pysrckeyword{for} jj \pysrckeyword{in} (\pysrcstring{'old'}, \pysrcstring{'blind'})
\pysrcmore{... }            \pysrckeyword{for} nn \pysrckeyword{in} (\pysrcstring{'men'}, \pysrcstring{'mice'}){]}
\pysrcoutput{{[}('two', 'old', 'men'), ('two', 'old', 'mice'), ('two', 'blind', 'men'),}
\pysrcoutput{ ('two', 'blind', 'mice'), ('three', 'old', 'men'), ('three', 'old', 'mice'),}
\pysrcoutput{ ('three', 'blind', 'men'), ('three', 'blind', 'mice'){]}}}\end{alltt}

{\noindent}
The above example contains three independent \texttt{\pysrckeyword{for}} loops.
These loops have no variables in common, and we could have put them in any order.
We can also have \index{nested loops}\textbf{nested loops} with shared variables.
The next example iterates over all sentences in a
section of the Brown Corpus, and for each sentence, iterates over each word.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }{[}word \pysrckeyword{for} word \pysrckeyword{in} nltk.corpus.brown.words(categories=\pysrcstring{'a'})
\pysrcmore{... }    \pysrckeyword{if} len(word) == 17{]}
\pysrcoutput{{[}'September-October', 'Sheraton-Biltmore', 'anti-organization',}
\pysrcoutput{ 'anti-organization', 'Washington-Oregon', 'York-Pennsylvania',}
\pysrcoutput{ 'misunderstandings', 'Sheraton-Biltmore', 'neo-stagnationist',}
\pysrcoutput{ 'cross-examination', 'bronzy-green-gold', 'Oh-the-pain-of-it',}
\pysrcoutput{ 'Secretary-General', 'Secretary-General', 'textile-importing',}
\pysrcoutput{ 'textile-exporting', 'textile-producing', 'textile-producing'{]}}}\end{alltt}

{\noindent} As you will see, the list comprehension in this example
contains a final \texttt{\pysrckeyword{if}} clause that allows us to filter out any words
that fail to meet the specified condition.

Another way to use loop variables is to ignore them!  This is
the standard method for building multidimensional structures.
For example, to build an array with \emph{m} rows and \emph{n} columns,
where each cell is a set, we would use a nested list comprehension,
as shown in line \href{\#2d-array}{①} below.  Observe that the loop variables
\texttt{i} and \texttt{j} are not used anywhere in the expressions preceding the
\texttt{\pysrckeyword{for}} clauses.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }m, n = 3, 7
\pysrcprompt{>{}>{}> }array = {[}{[}set() \pysrckeyword{for} i \pysrckeyword{in} range(n){]} \pysrckeyword{for} j \pysrckeyword{in} range(m){]}     ①
\pysrcprompt{>{}>{}> }array{[}2{]}{[}5{]}.add(\pysrcstring{'foo'})
\pysrcprompt{>{}>{}> }pprint.pprint(array)
\pysrcoutput{{[}{[}set({[}{]}), set({[}{]}), set({[}{]}), set({[}{]}), set({[}{]}), set({[}{]}), set({[}{]}){]},}
\pysrcoutput{ {[}set({[}{]}), set({[}{]}), set({[}{]}), set({[}{]}), set({[}{]}), set({[}{]}), set({[}{]}){]},}
\pysrcoutput{ {[}set({[}{]}), set({[}{]}), set({[}{]}), set({[}{]}), set({[}{]}), set({[}'foo'{]}), set({[}{]}){]}{]}}}\end{alltt}

Sometimes we use a list comprehension as part of a larger aggregation task.
In the following example we calculate the average length of words in part of the Brown Corpus.
Notice that we don't bother storing the list comprehension in a temporary
variable, but use it directly as an argument to the \texttt{average{\linebreak[0]}({\linebreak[0]})} function.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} numpy \pysrckeyword{import} average
\pysrcprompt{>{}>{}> }average({[}len(word) \pysrckeyword{for} word \pysrckeyword{in} nltk.corpus.brown.words(categories=\pysrcstring{'a'}){]})
\pysrcoutput{4.40154543827}}\end{alltt}

Now that we have reviewed the sequence types, we have one more fundamental data type
to revisit.


%___________________________________________________________________________

\subsection{Dictionaries}
\label{dictionaries}

As you have already seen, the dictionary data type can be used in a variety
of language processing tasks (e.g. \href{\#getting-organized}{Section 1.7}).
However, we have only scratched the surface.
Dictionaries have many more applications than you might have imagined.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

The dictionary data type is often known by the name
\index{associative array}\textbf{associative array}.  A normal array maps from integers (the keys)
to arbitrary data types (the values), while an associative array places
no such constraint on keys.  Keys can be strings, tuples, or other
more deeply nested structure.  Python places the constraint that
keys must be \index{immutable}\textit{immutable}.
\end{boxedminipage}
\end{sffamily}
\end{center}

Let's begin by comparing dictionaries with tuples.  Tuples allow
\docutilsroleemphasis{access by position}; to access the part-of-speech of the following
lexical entry we just have to know it is found at index position \texttt{1}.
However, dictionaries allow \docutilsroleemphasis{access by name}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }entry{\_}tuple = (\pysrcstring{'turned'}, \pysrcstring{'VBD'}, {[}\pysrcstring{'t3:nd'}, \pysrcstring{'t3`nd'}{]})
\pysrcprompt{>{}>{}> }entry{\_}tuple{[}1{]}
\pysrcoutput{'VBD'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }entry{\_}dict = {\{}\pysrcstring{'lexeme'}:\pysrcstring{'turned'}, \pysrcstring{'pos'}:\pysrcstring{'VBD'}, \pysrcstring{'pron'}:{[}\pysrcstring{'t3:nd'}, \pysrcstring{'t3`nd'}{]}{\}}
\pysrcprompt{>{}>{}> }entry{\_}dict{[}\pysrcstring{'pos'}{]}
\pysrcoutput{'VBD'}}\end{alltt}

In this case, dictionaries are little more than a convenience.  We can
even simulate access by name using well-chosen constants, e.g.:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }LEXEME = 0; POS = 1; PRON = 2
\pysrcprompt{>{}>{}> }entry{\_}tuple{[}POS{]}
\pysrcoutput{'VBD'}}\end{alltt}

This method works when there is a closed set of keys and the keys are known
in advance.  Dictionaries come into their own when we are mapping from
an open set of keys, which happens when the keys are drawn from an unrestricted
vocabulary or where they are generated by some procedure.  \href{\#mystery}{Listing 5.3}
illustrates the first of these.  The function \texttt{mystery{\linebreak[0]}({\linebreak[0]})}
begins by initializing a dictionary called \texttt{groups}, then populates it
with words.  We leave it as an exercise for the reader to work out what
this function computes.  For now, it's enough to note that the keys of
this dictionary are an open set, and it would not be feasible to use
a integer keys, as would be required if we used lists or tuples for the
representation.
% TODO: discussion ''.join() vs join() 
\hypertarget{mystery}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{mystery}(input):
    groups = {\{}{\}}
    \pysrckeyword{for} word \pysrckeyword{in} input:
        key = \pysrcstring{' '}.join(sorted(list(word)), \pysrcstring{'{}'})
        \pysrckeyword{if} key \pysrckeyword{not} \pysrckeyword{in} groups:               ①
            groups{[}key{]} = set()             ②
        groups{[}key{]}.add(word)               ③
    return sorted(\pysrcstring{' '}.join(sorted(v)) \pysrckeyword{for} v \pysrckeyword{in} groups.values() \pysrckeyword{if} len(v) > 1)}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }words = nltk.corpus.words.words()
\pysrcprompt{>{}>{}> }\pysrckeyword{print} mystery(words)                         }\end{alltt}
\caption{Mystery program}
\vspace{1ex}
\hrule
\end{pylisting}

\href{\#mystery}{Listing 5.3} illustrates two important idioms, which we already
touched on in \href{\#chap-programming}{Chapter 1}.  First, dictionary keys
are unique; in order to store multiple items in a single entry we
define the value to be a list or a set, and simply update the value each
time we want to store another item (line \href{\#mystery-update}{③}).  Second, if a key does
not yet exist in a dictionary (line \href{\#mystery-key}{①}) we must explicitly add it
and give it an initial value (line \href{\#mystery-add}{②}).

The second important use of dictionaries is for mappings that involve \index{compound keys}\textbf{compound keys}.
Suppose we want to categorize a series of linguistic observations according
to two or more properties.  We can combine the properties using a tuple
and build up a dictionary in the usual way, as exemplified in \href{\#compound-keys}{Listing 5.4}.
\hypertarget{compound-keys}{}
\begin{pylisting}
\begin{alltt}\small\textbf{attachment = nltk.defaultdict(\pysrckeyword{lambda}:{[}0,0{]})
V, N = 0, 1
\pysrckeyword{for} entry \pysrckeyword{in} nltk.corpus.ppattach.attachments(\pysrcstring{'training'}):
    key = entry.verb, entry.prep
    \pysrckeyword{if} entry.attachment == \pysrcstring{'V'}:
        attachment{[}key{]}{[}V{]} += 1
    \pysrckeyword{else}:
        attachment{[}key{]}{[}N{]} += 1}\end{alltt}
\caption{Illustration of compound keys}
\vspace{1ex}
\hrule
\end{pylisting}


%___________________________________________________________________________

\subsection{Exercises}
\label{id39}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Find out more about sequence objects using Python's help facility.
In the interpreter, type \texttt{help{\linebreak[0]}({\linebreak[0]}str{\linebreak[0]})}, \texttt{help{\linebreak[0]}({\linebreak[0]}list{\linebreak[0]})}, and \texttt{help{\linebreak[0]}({\linebreak[0]}tuple{\linebreak[0]})}.
This will give you a full list of the functions supported by each type.
Some functions have special names flanked with underscore; as the
help documentation shows, each such function corresponds to something
more familiar.  For example \texttt{x{\linebreak[0]}.{\linebreak[0]}{\_}{\_}getitem{\_}{\_}{\linebreak[0]}({\linebreak[0]}y{\linebreak[0]})} is just a long-winded
way of saying \texttt{x{\linebreak[0]}{[}{\linebreak[0]}y{\linebreak[0]}{]}}.

\item {} 
☼ Identify three operations that can be performed on both tuples
and lists.  Identify three list operations that cannot be performed on
tuples.  Name a context where using a list instead of a tuple generates
a Python error.

\item {} 
☼ Find out how to create a tuple consisting of a single item.
There are at least two ways to do this.

\item {} 
☼ Create a list \texttt{words{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}is{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}NLP{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}fun{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}?{\linebreak[0]}'}{]}}.  Use
a series of assignment statements (e.g. \texttt{words{\linebreak[0]}{[}{\linebreak[0]}1{\linebreak[0]}{]}{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}words{\linebreak[0]}{[}{\linebreak[0]}2{\linebreak[0]}{]}})
and a temporary variable \texttt{tmp} to transform this list into the
list \texttt{{[}\pysrcstring{'{\linebreak[0]}NLP{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}is{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}fun{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}!{\linebreak[0]}'}{]}}.  Now do the same transformation
using tuple assignment.

\item {} 
☼ Does the method for creating a sliding window of n-grams
behave correctly for the two limiting cases: \docutilsrolemath{n} = 1, and \docutilsrolemath{n} = \texttt{len{\linebreak[0]}({\linebreak[0]}sent{\linebreak[0]})}?

\item {} 
◑ Create a list of words and store it in a variable \texttt{sent1}.
Now assign \texttt{sent2{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}sent1}.  Modify one of the items in \texttt{sent1}
and verify that \texttt{sent2} has changed.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Now try the same exercise but instead assign \texttt{sent2{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}sent1{\linebreak[0]}{[}{\linebreak[0]}:{\linebreak[0]}{]}}.
Modify \texttt{sent1} again and see what happens to \texttt{sent2}.  Explain.

\item {} 
Now define \texttt{text1} to be a list of lists of strings (e.g. to
represent a text consisting of multiple sentences.  Now assign
\texttt{text2{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}text1{\linebreak[0]}{[}{\linebreak[0]}:{\linebreak[0]}{]}}, assign a new value to one of the words,
e.g. \texttt{text1{\linebreak[0]}{[}{\linebreak[0]}1{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}1{\linebreak[0]}{]}{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} \pysrcstring{'{\linebreak[0]}Monty{\linebreak[0]}'}}.  Check what this did to \texttt{text2}.
Explain.

\item {} 
Load Python's \texttt{deepcopy{\linebreak[0]}({\linebreak[0]})} function (i.e. \texttt{\pysrckeyword{from} \pysrcbuiltin{copy} \pysrckeyword{import} {\linebreak[0]}deepcopy}),
consult its documentation, and test that it makes a fresh copy of any
object.

\end{list}

\item {} 
◑ Write code that starts with a string of words and
results in a new string consisting of the same words, but where
the first word swaps places with the second, and so on. For
example, \texttt{\pysrcstring{'{\linebreak[0]}the{\linebreak[0]} {\linebreak[0]}cat{\linebreak[0]} {\linebreak[0]}sat{\linebreak[0]} {\linebreak[0]}on{\linebreak[0]} {\linebreak[0]}the{\linebreak[0]} {\linebreak[0]}mat{\linebreak[0]}'}} will be converted into \texttt{\pysrcstring{'{\linebreak[0]}cat}
\pysrcstring{the{\linebreak[0]} {\linebreak[0]}on{\linebreak[0]} {\linebreak[0]}sat{\linebreak[0]} {\linebreak[0]}mat{\linebreak[0]} {\linebreak[0]}the{\linebreak[0]}'}}.

\item {} 
◑ Initialize an \emph{n}-by-\emph{m} list of lists of empty strings using list
multiplication, e.g. \texttt{word{\_}table{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}{[}\pysrcstring{'{\linebreak[0]}'}{]}{\linebreak[0]} {\linebreak[0]}*{\linebreak[0]} {\linebreak[0]}n{\linebreak[0]}{]}{\linebreak[0]} {\linebreak[0]}*{\linebreak[0]} {\linebreak[0]}m}.  What happens
when you set one of its values, e.g. \texttt{word{\_}table{\linebreak[0]}{[}{\linebreak[0]}1{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}2{\linebreak[0]}{]}{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} \pysrcstring{"{\linebreak[0]}hello{\linebreak[0]}"}}?
Explain why this happens.  Now write an expression using \texttt{range{\linebreak[0]}({\linebreak[0]})}
to construct a list of lists, and show that it does not have this problem.

\item {} 
◑ Write code to initialize a two-dimensional array of sets called
\texttt{word{\_}vowels} and process a list of words, adding each
word to \texttt{word{\_}vowels{\linebreak[0]}{[}{\linebreak[0]}l{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}v{\linebreak[0]}{]}} where \texttt{l} is the length of the word and \texttt{v} is
the number of vowels it contains.

\item {} 
◑ Write code that builds a dictionary of dictionaries of sets.

\item {} 
◑ Use \texttt{sorted{\linebreak[0]}({\linebreak[0]})} and \texttt{set{\linebreak[0]}({\linebreak[0]})} to get a sorted list of tags used in the Brown
corpus, removing duplicates.

\item {} 
◑ Read up on Gematria, a method for assigning numbers to words, and for
mapping between words having the same number to discover the hidden meaning of
texts (\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}en{\linebreak[0]}.{\linebreak[0]}wikipedia{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}wiki{\linebreak[0]}/{\linebreak[0]}Gematria}, \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}essenes{\linebreak[0]}.{\linebreak[0]}net{\linebreak[0]}/{\linebreak[0]}gemcal{\linebreak[0]}.{\linebreak[0]}htm}).
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Write a function \texttt{gematria{\linebreak[0]}({\linebreak[0]})} that sums the numerical values of
the letters of a word, according to the letter values in \texttt{letter{\_}vals}:
\begin{description}
\item[{letter{\_}vals = {\{}'a':1, 'b':2, 'c':3, 'd':4, 'e':5, 'f':80, 'g':3, 'h':8,}] \leavevmode 
'i':10, 'j':10, 'k':20, 'l':30, 'm':40, 'n':50, 'o':70, 'p':80, 'q':100,
'r':200, 's':300, 't':400, 'u':6, 'v':6, 'w':800, 'x':60, 'y':10, 'z':7{\}}

\end{description}

\item {} 
Use the method from \href{\#mystery}{Listing 5.3} to index English words according to their values.

\item {} 
Process a corpus (e.g. \texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}state{\_}union}) and for each document, count how
many of its words have the number 666.

\item {} 
Write a function \texttt{decode{\linebreak[0]}({\linebreak[0]})} to process a text, randomly replacing words with
their Gematria equivalents, in order to discover the ``hidden meaning'' of the text.

\end{list}

\item {} 
★ Extend the example in \href{\#compound-keys}{Listing 5.4} in the following ways:
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Define two sets \texttt{verbs} and \texttt{preps}, and add each verb and preposition
as they are encountered.  (Note that you can add an item to a set without
bothering to check whether it is already present.)

\item {} 
Create nested loops to display the results, iterating over verbs and
prepositions in sorted order.  Generate one line of output per verb,
listing prepositions and attachment ratios as follows:
\texttt{raised{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}about{\linebreak[0]} {\linebreak[0]}0{\linebreak[0]}:{\linebreak[0]}3{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}at{\linebreak[0]} {\linebreak[0]}1{\linebreak[0]}:{\linebreak[0]}0{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}by{\linebreak[0]} {\linebreak[0]}9{\linebreak[0]}:{\linebreak[0]}0{\linebreak[0]},{\linebreak[0]} \pysrckeyword{for} {\linebreak[0]}3{\linebreak[0]}:{\linebreak[0]}6{\linebreak[0]},{\linebreak[0]} \pysrckeyword{from} {\linebreak[0]}5{\linebreak[0]}:{\linebreak[0]}0{\linebreak[0]},{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}5{\linebreak[0]}:{\linebreak[0]}5{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}.}

\item {} 
We used a tuple to represent a compound key consisting of two strings.
However, we could have simply concatenated the strings, e.g.
\texttt{key{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}verb{\linebreak[0]} {\linebreak[0]}+{\linebreak[0]} \pysrcstring{"{\linebreak[0]}:{\linebreak[0]}"} {\linebreak[0]}+{\linebreak[0]} {\linebreak[0]}prep}, resulting in a simple string key.
Why is it better to use tuples for compound keys?

\end{list}

\end{list}


%___________________________________________________________________________

\section{Presenting Results}
\label{presenting-results}

Often we write a program to report a single datum, such as a particular element
in a corpus that meets some complicated criterion, or a single summary statistic
such as a word-count or the performance of a tagger.  More often, we write a program
to produce a structured result, such as a tabulation of numbers or linguistic forms,
or a reformatting of the original data.  When the results to be presented are linguistic,
textual output is usually the most natural choice.  However, when the results are numerical,
it may be preferable to produce graphical output.  In this section you will learn about
a variety of ways to present program output.


%___________________________________________________________________________

\subsection{Strings and Formats}
\label{strings-and-formats}

We have seen that there are two ways to display the contents of an object:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }word = \pysrcstring{'cat'}
\pysrcprompt{>{}>{}> }sentence = \pysrcstring{"{}"{}"hello}
\pysrcmore{... }\pysrcstring{world"{}"{}"}
\pysrcprompt{>{}>{}> }\pysrckeyword{print} word
\pysrcoutput{cat}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{print} sentence
\pysrcoutput{hello}
\pysrcoutput{world}
\pysrcoutput{}\pysrcprompt{>{}>{}> }word
\pysrcoutput{'cat'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }sentence
\pysrcoutput{'hello{\textbackslash}nworld'}}\end{alltt}

The \texttt{\pysrckeyword{print}} command yields Python's attempt to produce the most human-readable form of an object.
The second method — naming the variable at a prompt — shows us a string
that can be used to recreate this object.  It is important to keep in mind that both of
these are just strings, displayed for the benefit of you, the user.  They do not give
us any clue as to the actual internal representation of the object.

There are many other useful ways to display an object as a string of
characters.  This may be for the benefit of a human reader, or because
we want to \index{export}\textbf{export} our data to a particular file format for use
in an external program.

Formatted output typically contains a combination of variables and
pre-specified strings, e.g. given a dictionary \texttt{wordcount}
consisting of words and their frequencies we could do:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }wordcount = {\{}\pysrcstring{'cat'}:3, \pysrcstring{'dog'}:4, \pysrcstring{'snake'}:1{\}}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} sorted(wordcount):
\pysrcmore{... }    \pysrckeyword{print} word, \pysrcstring{'->'}, wordcount{[}word{]}, \pysrcstring{';'},
\pysrcoutput{cat -> 3 ; dog -> 4 ; snake -> 1 ;}}\end{alltt}

Apart from the problem of unwanted whitespace, print statements
that contain alternating variables and constants can be difficult to read and
maintain.  A better solution is to use formatting strings:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} sorted(wordcount):
\pysrcmore{... }   \pysrckeyword{print} \pysrcstring{'{\%}s->{\%}d;'} {\%} (word, wordcount{[}word{]}),
\pysrcoutput{cat->3; dog->4; snake->1;}}\end{alltt}
% TODO: describe textwrap package 
\hypertarget{sec-lining-things-up}{}

%___________________________________________________________________________

\subsection{Lining Things Up}
\label{lining-things-up}
\label{sec-lining-things-up}

So far our formatting strings have contained specifications of fixed width, such
as \texttt{{\%}{\linebreak[0]}6s}, a string that is padded to width 6 and right-justified.
We can include a minus sign to make it left-justified.
In case we don't know in advance how wide a displayed value should be,
the width value can be replaced with a star in the formatting string,
then specified using a variable:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrcstring{'{\%}6s'} {\%} \pysrcstring{'dog'}
\pysrcoutput{'   dog'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'{\%}-6s'} {\%} \pysrcstring{'dog'}
\pysrcoutput{'dog   '}
\pysrcoutput{}\pysrcprompt{>{}>{}> }width = 6
\pysrcprompt{>{}>{}> }\pysrcstring{'{\%}-*s'} {\%} (width, \pysrcstring{'dog'})
\pysrcoutput{'dog   '}}\end{alltt}

Other control characters are used for decimal integers and floating point numbers.
Since the percent character \texttt{{\%}} has a special interpretation in formatting strings,
we have to precede it with another \texttt{{\%}} to get it in the output:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrcstring{"accuracy for {\%}d words: {\%}2.4f{\%}{\%}"} {\%} (9375, 100.0 * 3205/9375)
\pysrcoutput{'accuracy for 9375 words: 34.1867{\%}'}}\end{alltt}

An important use of formatting strings is for tabulating data.
The program in \href{\#modal-tabulate}{Listing 5.5} iterates over five genres of the Brown Corpus.
For each token having the \texttt{md} tag we increment a count.  To do this we
have used \texttt{ConditionalFreqDist{\linebreak[0]}({\linebreak[0]})}, where the condition is the current genre
and the event is the modal, i.e. this constructs a frequency distribution of
the modal verbs in each genre.
Line \href{\#modals-of-interest}{①}
identifies a small set of modals of interest, and calls
the function \texttt{tabulate{\linebreak[0]}({\linebreak[0]})} that
processes the data structure to output the required counts.
Note that we have been careful to separate the language processing
from the tabulation of results.
\hypertarget{modal-tabulate}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{count{\_}words{\_}by{\_}tag}(t, genres):
    cfdist = nltk.ConditionalFreqDist()
    \pysrckeyword{for} genre \pysrckeyword{in} genres:
        \pysrckeyword{for} (word,tag) \pysrckeyword{in} nltk.corpus.brown.tagged{\_}words(categories=genre):
            \pysrckeyword{if} tag == t:
                 cfdist{[}genre{]}.inc(word.lower())
    return cfdist

\pysrckeyword{def} \pysrcdefname{tabulate}(cfdist, words):
    \pysrckeyword{print} \pysrcstring{'Genre '}, \pysrcstring{' '}.join({[}(\pysrcstring{'{\%}6s'} {\%} w) \pysrckeyword{for} w \pysrckeyword{in} words{]})
    \pysrckeyword{for} genre \pysrckeyword{in} sorted(cfdist.conditions()):           \pysrccomment{{\#} for each genre}
        \pysrckeyword{print} \pysrcstring{'{\%}-6s'} {\%} genre,                           \pysrccomment{{\#} print row heading}
        \pysrckeyword{for} w \pysrckeyword{in} words:                                 \pysrccomment{{\#} for each word}
            \pysrckeyword{print} \pysrcstring{'{\%}6d'} {\%} cfdist{[}genre{]}{[}w{]},             \pysrccomment{{\#} print table cell}
        \pysrckeyword{print}                                           \pysrccomment{{\#} end the row}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }genres = {[}\pysrcstring{'a'}, \pysrcstring{'d'}, \pysrcstring{'e'}, \pysrcstring{'h'}, \pysrcstring{'n'}{]}
\pysrcprompt{>{}>{}> }cfdist = count{\_}words{\_}by{\_}tag(\pysrcstring{'MD'}, genres)
\pysrcprompt{>{}>{}> }modals = {[}\pysrcstring{'can'}, \pysrcstring{'could'}, \pysrcstring{'may'}, \pysrcstring{'might'}, \pysrcstring{'must'}, \pysrcstring{'will'}{]}   ①
\pysrcprompt{>{}>{}> }tabulate(cfdist, modals)
\pysrcoutput{Genre     can  could    may  might   must   will}
\pysrcoutput{a          94     86     66     36     50    387}
\pysrcoutput{d          84     59     79     12     54     64}
\pysrcoutput{e         273     59    130     22     83    259}
\pysrcoutput{h         115     37    152     13     99    237}
\pysrcoutput{n          48    154      6     58     27     48}}\end{alltt}
\caption{Frequency of Modals in Different Sections of the Brown Corpus}
\vspace{1ex}
\hrule
\end{pylisting}

There are some interesting patterns in the table produced by
\href{\#modal-tabulate}{Listing 5.5}.  For instance,
compare row d (government literature) with row n (adventure literature);
the former is dominated by the use of \texttt{can{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}may{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}must{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}will} while
the latter is characterized by the use of \texttt{could} and \texttt{might}.
With some further work it might be possible to guess the genre of a
new text automatically, simply using information about the distribution of modal
verbs.

Our next example, in \href{\#concordance}{Listing 5.6}, generates a concordance display.
We use the left/right justification of strings and the variable width
to get vertical alignment of a variable-width window.

{[}TODO: explain ValueError exception{]}
% TODO: the following would be better if the corpus was an argument, and if
% brown.raw() behaved like other raw() methods and generated words not lists 
\hypertarget{concordance}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{concordance}(word, context):
    \pysrcstring{"Generate a concordance for the word with the specified context window"}
    \pysrckeyword{for} sent \pysrckeyword{in} nltk.corpus.brown.sents(categories=\pysrcstring{'a'}):
        try:
            pos = sent.index(word)
            left = \pysrcstring{' '}.join(sent{[}:pos{]})
            right = \pysrcstring{' '}.join(sent{[}pos+1:{]})
            \pysrckeyword{print} \pysrcstring{'{\%}*s {\%}s {\%}-*s'} {\%}{\textbackslash}
                (context, left{[}-context:{]}, word, context, right{[}:context{]})
        \pysrckeyword{except} ValueError:
            \pysrckeyword{pass}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }concordance(\pysrcstring{'line'}, 32)
\pysrcoutput{ce , is today closer to the NATO line .}
\pysrcoutput{n more activity across the state line in Massachusetts than in Rhode I}
\pysrcoutput{ , gained five yards through the line and then uncorked a 56-yard touc}
\pysrcoutput{                 `{}` Our interior line and out linebackers played excep}
\pysrcoutput{k then moved Cooke across with a line drive to left .}
\pysrcoutput{chal doubled down the rightfield line and Cooke singled off Phil Shart}
\pysrcoutput{              -{}- Billy Gardner's line double , which just eluded the d}
\pysrcoutput{           -{}- Nick Skorich , the line coach for the football champion}
\pysrcoutput{                     Maris is in line for a big raise .}
\pysrcoutput{uld be impossible to work on the line until then because of the large}
\pysrcoutput{         Murray makes a complete line of ginning equipment except for}
\pysrcoutput{    The company sells a complete line of gin machinery all over the co}
\pysrcoutput{tter Co. of Sherman makes a full line of gin machinery and equipment .}
\pysrcoutput{fred E. Perlman said Tuesday his line would face the threat of bankrup}
\pysrcoutput{ sale of property disposed of in line with a plan of liquidation .}
\pysrcoutput{ little effort spice up any chow line .}
\pysrcoutput{es , filed through the cafeteria line .}
\pysrcoutput{l be particularly sensitive to a line between first and second class c}
\pysrcoutput{A skilled worker on the assembly line , for example , earns {\$}37 a week}}\end{alltt}
\caption{Simple Concordance Display}
\vspace{1ex}
\hrule
\end{pylisting}


%___________________________________________________________________________

\subsection{Writing Results to a File}
\label{writing-results-to-a-file}

We have seen how to read text from files (\href{\#reading-files}{Section 2.2.1}).
It is often useful to write output to files as well.  The following
code opens a file \texttt{output{\linebreak[0]}.{\linebreak[0]}txt} for writing, and saves the program
output to the file.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }file = open(\pysrcstring{'output.txt'}, \pysrcstring{'w'})
\pysrcprompt{>{}>{}> }words = set(nltk.corpus.genesis.words(\pysrcstring{'english-kjv.txt'}))
\pysrcprompt{>{}>{}> }\pysrckeyword{for} word \pysrckeyword{in} sorted(words):
\pysrcmore{... }    file.write(word + \pysrcstring{"{\textbackslash}n"})}\end{alltt}

When we write non-text data to a file we must convert it to a string first.
We can do this conversion using formatting strings, as we saw above.
We can also do it using Python's backquote notation, which converts any
object into a string.  Let's write the total number of words to our
file, before closing it.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }len(words)
\pysrcoutput{2789}
\pysrcoutput{}\pysrcprompt{>{}>{}> }`len(words)`
\pysrcoutput{'2789'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }file.write(`len(words)` + \pysrcstring{"{\textbackslash}n"})
\pysrcprompt{>{}>{}> }file.close()}\end{alltt}
\hypertarget{sec-graphics}{}

%___________________________________________________________________________

\subsection{Graphical Presentation}
\label{graphical-presentation}
\label{sec-graphics}

So far we have focused on textual presentation and the use of formatted print
statements to get output lined up in columns.  It is often very useful to display
numerical data in graphical form, since this often makes it easier to detect
patterns.  For example, in \href{\#modal-tabulate}{Listing 5.5} we saw a table of numbers
showing the frequency of particular modal verbs in the Brown Corpus, classified
by genre.  In \href{\#modal-plot}{Listing 5.7} we present the same information in graphical
format.  The output is shown in \href{\#modal-genre}{Figure 5.3} (a color figure in the online version).
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

\href{\#modal-plot}{Listing 5.7} uses the PyLab package which supports sophisticated
plotting functions with a MATLAB-style interface.  For more information about
this package please see \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}matplotlib{\linebreak[0]}.{\linebreak[0]}sourceforge{\linebreak[0]}.{\linebreak[0]}net{\linebreak[0]}/}.
\end{boxedminipage}
\end{sffamily}
\end{center}
\hypertarget{modal-plot}{}
\begin{pylisting}
\begin{alltt}\small\textbf{colors = \pysrcstring{'rgbcmyk'} \pysrccomment{{\#} red, green, blue, cyan, magenta, yellow, black}
\pysrckeyword{def} \pysrcdefname{bar{\_}chart}(categories, words, counts):
    \pysrcstring{"Plot a bar chart showing counts for each word by category"}
    \pysrckeyword{import} pylab
    ind = pylab.arange(len(words))
    width = 1.0 / (len(categories) + 1)
    bar{\_}groups = {[}{]}
    \pysrckeyword{for} c \pysrckeyword{in} range(len(categories)):
        bars = pylab.bar(ind+c*width, counts{[}categories{[}c{]}{]}, width, color=colors{[}c {\%} len(colors){]})
        bar{\_}groups.append(bars)
    pylab.xticks(ind+width, words)
    pylab.legend({[}b{[}0{]} \pysrckeyword{for} b \pysrckeyword{in} bar{\_}groups{]}, categories, loc=\pysrcstring{'upper left'})
    pylab.ylabel(\pysrcstring{'Frequency'})
    pylab.title(\pysrcstring{'Frequency of Six Modal Verbs by Genre'})
    pylab.show()}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }genres = {[}\pysrcstring{'a'}, \pysrcstring{'d'}, \pysrcstring{'e'}, \pysrcstring{'h'}, \pysrcstring{'n'}{]}
\pysrcprompt{>{}>{}> }cfdist = count{\_}words{\_}by{\_}tag(\pysrcstring{'MD'}, genres)
\pysrcprompt{>{}>{}> }modals = {[}\pysrcstring{'can'}, \pysrcstring{'could'}, \pysrcstring{'may'}, \pysrcstring{'might'}, \pysrcstring{'must'}, \pysrcstring{'will'}{]}
\pysrcprompt{>{}>{}> }counts = {\{}{\}}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} genre \pysrckeyword{in} genres:
\pysrcmore{... }    counts{[}genre{]} = {[}cfdist{[}genre{]}{[}word{]} \pysrckeyword{for} word \pysrckeyword{in} modals{]}
\pysrcprompt{>{}>{}> }bar{\_}chart(genres, modals, counts)}\end{alltt}
\caption{Frequency of Modals in Different Sections of the Brown Corpus}
\vspace{1ex}
\hrule
\end{pylisting}
\hypertarget{modal-genre}{}\begin{figure}[htbp]
\centering

\scalebox{0.128571}{\includegraphics{../images/modal_genre.png}}
\caption{Bar Chart Showing Frequency of Modals in Different Sections of Brown Corpus}\end{figure}

From the bar chart it is immediately obvious that \docutilsroleexample{may} and \docutilsroleexample{must} have
almost identical relative frequencies.  The same goes for \docutilsroleexample{could} and \docutilsroleexample{might}.


%___________________________________________________________________________

\subsection{Exercises}
\label{id40}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Write code that removes whitespace at the beginning and end of a
string, and normalizes whitespace between words to be a single
space character.
\setcounter{listcnt1}{0}
\begin{list}{\arabic{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
do this task using \texttt{split{\linebreak[0]}({\linebreak[0]})} and \texttt{join{\linebreak[0]}({\linebreak[0]})}

\item {} 
do this task using regular expression substitutions

\end{list}

\item {} 
☼ What happens when the formatting strings \texttt{{\%}{\linebreak[0]}6s} and \texttt{{\%}{\linebreak[0]}-{\linebreak[0]}6s}
are used to display strings that are longer than six characters?

\item {} 
☼ We can use a dictionary to specify the values to be
substituted into a formatting string.  Read Python's library
documentation for formatting strings
(\href{http://docs.python.org/lib/typesseq-strings.html}{http://docs.python.org/lib/typesseq-strings.html}),
and use this method to display today's date in two
different formats.

\item {} 
◑ \href{\#baseline-tagger}{Listing 3.3} in \href{\#chap-tag}{Chapter 3} plotted a curve showing
change in the performance of a lookup tagger as the model size was increased.
Plot the performance curve for a unigram tagger, as the amount of training
data is varied.

\end{list}
\hypertarget{sec-functions}{}

%___________________________________________________________________________

\section{Functions}
\label{functions}
\label{sec-functions}

Once you have been programming for a while, you will find that you need
to perform a task that you have done in the past.  In fact, over time,
the number of completely novel things you have to do in creating a program
decreases significantly.  Half of the work may involve simple tasks that
you have done before.  Thus it is important for your code to be \docutilsroleemphasis{re-usable}.
One effective way to do this is to abstract commonly used sequences of steps
into a \index{function}\textbf{function}, as we briefly saw in \href{\#chap-programming}{Chapter 1}.

For example, suppose we find that we often want to read text from an HTML file.
This involves several steps: opening the file, reading it in, normalizing
whitespace, and stripping HTML markup.  We can collect these steps into a
function, and give it a name such as \texttt{get{\_}text{\linebreak[0]}({\linebreak[0]})}:
\hypertarget{get-text}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{import} re
\pysrckeyword{def} \pysrcdefname{get{\_}text}(file):
    \pysrcstring{"{}"{}"Read text from a file, normalizing whitespace}
\pysrcstring{    and stripping HTML markup."{}"{}"}
    text = open(file).read()
    text = re.sub(\pysrcstring{'{\textbackslash}s+'}, \pysrcstring{' '}, text)
    text = re.sub(r\pysrcstring{'<.*?>'}, \pysrcstring{' '}, text)
    return text}\end{alltt}
\caption{Read text from a file}
\vspace{1ex}
\hrule
\end{pylisting}

Now, any time we want to get cleaned-up text from an HTML file, we can just call
\texttt{get{\_}text{\linebreak[0]}({\linebreak[0]})} with the name of the file as its only argument.  It will return
a string, and we can assign this to a variable, e.g.:
\texttt{contents{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}get{\_}text{\linebreak[0]}(\pysrcstring{"{\linebreak[0]}test{\linebreak[0]}.{\linebreak[0]}html{\linebreak[0]}"})}.  Each time we want to use this series of
steps we only have to call the function.

Notice that a function definition consists of the keyword \texttt{def} (short for ``define''), followed
by the function name, followed by a sequence of parameters enclosed in parentheses, then
a colon.  The following lines contain an indented block of code, the \index{function body}\textbf{function body}.

Using functions has the benefit of saving space in our program.  More
importantly, our choice of name for the function helps make the program \emph{readable}.
In the case of the above example, whenever our program needs to read cleaned-up
text from a file we don't have to clutter the program with four lines of code, we
simply need to call \texttt{get{\_}text{\linebreak[0]}({\linebreak[0]})}.  This naming helps to provide some ``semantic
interpretation'' — it helps a reader of our program to see what the program ``means''.

Notice that the above function definition contains a string.  The first string inside
a function definition is called a \index{docstring}\textbf{docstring}.  Not only does it document the
purpose of the function to someone reading the code, it is accessible to a programmer
who has loaded the code from a file:
\begin{quote}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }help(get{\_}text)
\pysrcoutput{Help on function get{\_}text:}}\end{alltt}
\begin{description}
\item[{get{\_}text(file)}] \leavevmode 
Read text from a file, normalizing whitespace
and stripping HTML markup.

\end{description}
\end{quote}

We have seen that functions help to make our work reusable and readable.  They
also help make it \emph{reliable}.  When we re-use code that has already been developed
and tested, we can be more confident that it handles a variety of cases correctly.
We also remove the risk that we forget some important step, or introduce a bug.
The program that calls our function also has increased reliability.  The author
of that program is dealing with a shorter program, and its components behave
transparently.
\begin{itemize}
\item {} 
{[}More: overview of section{]}

\end{itemize}


%___________________________________________________________________________

\subsection{Function Arguments}
\label{function-arguments}
\begin{itemize}
\item {} 
multiple arguments

\item {} 
named arguments

\item {} 
default values

\end{itemize}

Python is a \index{dynamically typed}\textbf{dynamically typed} language.  It does not force us to
declare the type of a variable when we write a program.  This feature
is often useful, as it permits us to define functions that are flexible
about the type of their arguments.  For example, a tagger might expect
a sequence of words, but it wouldn't care whether this sequence is expressed
as a list, a tuple, or an iterator.

However, often we want to write programs for later use by others, and want
to program in a defensive style, providing useful warnings when functions
have not been invoked correctly.  Observe that the \texttt{tag{\linebreak[0]}({\linebreak[0]})}
function in \href{\#tag1}{Listing 5.9} behaves sensibly for string arguments,
but that it does not complain when it is passed a dictionary.
\hypertarget{tag1}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{tag}(word):
    \pysrckeyword{if} word \pysrckeyword{in} {[}\pysrcstring{'a'}, \pysrcstring{'the'}, \pysrcstring{'all'}{]}:
        return \pysrcstring{'DT'}
    \pysrckeyword{else}:
        return \pysrcstring{'NN'}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tag(\pysrcstring{'the'})
\pysrcoutput{'DT'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }tag(\pysrcstring{'dog'})
\pysrcoutput{'NN'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }tag({\{}\pysrcstring{'lexeme'}:\pysrcstring{'turned'}, \pysrcstring{'pos'}:\pysrcstring{'VBD'}, \pysrcstring{'pron'}:{[}\pysrcstring{'t3:nd'}, \pysrcstring{'t3`nd'}{]}{\}})
\pysrcoutput{'NN'}}\end{alltt}
\caption{A tagger that tags anything}
\vspace{1ex}
\hrule
\end{pylisting}

It would be helpful if the author of this function took some extra steps to
ensure that the \texttt{word} parameter of the \texttt{tag{\linebreak[0]}({\linebreak[0]})} function is a string.
A naive approach would be to check the type of the argument and return
a diagnostic value, such as Python's special empty value, \texttt{None},
as shown in \href{\#tag2}{Listing 5.10}.
\hypertarget{tag2}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{tag}(word):
    \pysrckeyword{if} \pysrckeyword{not} type(word) \pysrckeyword{is} str:
        return None
    \pysrckeyword{if} word \pysrckeyword{in} {[}\pysrcstring{'a'}, \pysrcstring{'the'}, \pysrcstring{'all'}{]}:
        return \pysrcstring{'DT'}
    \pysrckeyword{else}:
        return \pysrcstring{'NN'}}\end{alltt}
\caption{A tagger that only tags strings}
\vspace{1ex}
\hrule
\end{pylisting}

However, this approach is dangerous because the calling program
may not detect the error, and the diagnostic return value may be
propagated to later parts of the program with unpredictable consequences.
A better solution is shown in \href{\#tag3}{Listing 5.11}.
\hypertarget{tag3}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{tag}(word):
    \pysrckeyword{if} \pysrckeyword{not} type(word) \pysrckeyword{is} str:
        raise ValueError, \pysrcstring{"argument to tag() must be a string"}
    \pysrckeyword{if} word \pysrckeyword{in} {[}\pysrcstring{'a'}, \pysrcstring{'the'}, \pysrcstring{'all'}{]}:
        return \pysrcstring{'DT'}
    \pysrckeyword{else}:
        return \pysrcstring{'NN'}}\end{alltt}
\caption{A tagger that generates an error message when not passed a string}
\vspace{1ex}
\hrule
\end{pylisting}

This produces an error that cannot be ignored, since it halts program execution.
Additionally, the error message is easy to interpret.  (We will see an even
better approach, known as ``duck typing'' in \href{\#chap-applied-programming}{Chapter 9}.)

Another aspect of defensive programming concerns the return statement of a function.
In order to be confident that all execution paths through a function lead to a
return statement, it is best to have a single return statement at the end of
the function definition.
This approach has a further benefit: it makes it more likely that the
function will only return a single type.
Thus, the following version of our \texttt{tag{\linebreak[0]}({\linebreak[0]})} function is safer:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{tag}(word):
\pysrcmore{... }    result = \pysrcstring{'NN'}                       \pysrccomment{{\#} default value, a string}
\pysrcmore{... }    \pysrckeyword{if} word \pysrckeyword{in} {[}\pysrcstring{'a'}, \pysrcstring{'the'}, \pysrcstring{'all'}{]}:     \pysrccomment{{\#} in certain cases...}
\pysrcmore{... }        result = \pysrcstring{'DT'}                   \pysrccomment{{\#}   overwrite the value}
\pysrcmore{... }    return result                       \pysrccomment{{\#} all paths end here}}\end{alltt}

A return statement can be used to pass multiple values back to the calling
program, by packing them into a tuple.
Here we define a function that returns a tuple
consisting of the average word length of a sentence, and the inventory
of letters used in the sentence.  It would have been clearer to write
two separate functions.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{proc{\_}words}(words):
\pysrcmore{... }    avg{\_}wordlen = sum(len(word) \pysrckeyword{for} word \pysrckeyword{in} words)/len(words)
\pysrcmore{... }    chars{\_}used = \pysrcstring{'{}'}.join(sorted(set(\pysrcstring{'{}'}.join(words))))
\pysrcmore{... }    return avg{\_}wordlen, chars{\_}used
\pysrcprompt{>{}>{}> }proc{\_}words({[}\pysrcstring{'Not'}, \pysrcstring{'a'}, \pysrcstring{'good'}, \pysrcstring{'way'}, \pysrcstring{'to'}, \pysrcstring{'write'}, \pysrcstring{'functions'}{]})
\pysrcoutput{(3, 'Nacdefginorstuwy')}}\end{alltt}
% EK: wouldn't it be better to use an example which isn't then open to criticism? 

Functions do not need to have a return statement at all.
Some functions do their work as a side effect, printing a result,
modifying a file, or updating the contents of a parameter to the function.
Consider the following three sort functions; the last approach is dangerous
because a programmer could use it without realizing that it had modified
its input.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{my{\_}sort1}(l):      \pysrccomment{{\#} good: modifies its argument, no return value}
\pysrcmore{... }    l.sort()
\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{my{\_}sort2}(l):      \pysrccomment{{\#} good: doesn't touch its argument, returns value}
\pysrcmore{... }    return sorted(l)
\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{my{\_}sort3}(l):      \pysrccomment{{\#} bad: modifies its argument and also returns it}
\pysrcmore{... }    l.sort()
\pysrcmore{... }    return l}\end{alltt}


%___________________________________________________________________________

\subsection{An Important Subtlety}
\label{an-important-subtlety}

Back in \href{\#sec-assignment}{Section 5.2.1} you saw that in Python, assignment works on values,
but that the value of a structured object is a reference to that object.  The same
is true for functions.  Python interprets function parameters as values (this is
known as \index{call-by-value}\textbf{call-by-value}).  Consider \href{\#call-by-value}{Listing 5.12}.
Function \texttt{set{\_}up{\linebreak[0]}({\linebreak[0]})} has two parameters,
both of which are modified inside the function.  We begin by assigning an empty string
to \texttt{w} and an empty dictionary to \texttt{p}.  After calling the function, \texttt{w} is unchanged,
while \texttt{p} is changed:
\hypertarget{call-by-value}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{set{\_}up}(word, properties):
    word = \pysrcstring{'cat'}
    properties{[}\pysrcstring{'pos'}{]} = \pysrcstring{'noun'}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }w = \pysrcstring{'{}'}
\pysrcprompt{>{}>{}> }p = {\{}{\}}
\pysrcprompt{>{}>{}> }set{\_}up(w, p)
\pysrcprompt{>{}>{}> }w
\pysrcoutput{'{}'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }p
\pysrcoutput{{\{}'pos': 'noun'{\}}}}\end{alltt}
\caption{}
\vspace{1ex}
\hrule
\end{pylisting}

{\noindent}
To understand why \texttt{w} was not changed, it is necessary to understand call-by-value.
When we called \texttt{set{\_}up{\linebreak[0]}({\linebreak[0]}w{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}p{\linebreak[0]})}, the value of \texttt{w} (an empty string) was assigned to
a new variable \texttt{word}.  Inside the function, the value of \texttt{word} was modified.
However, that had no effect on the external value of \texttt{w}.  This parameter passing is
identical to the following sequence of assignments:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }w = \pysrcstring{'{}'}
\pysrcprompt{>{}>{}> }word = w
\pysrcprompt{>{}>{}> }word = \pysrcstring{'cat'}
\pysrcprompt{>{}>{}> }w
\pysrcoutput{'{}'}}\end{alltt}

{\noindent}
In the case of the structured object, matters are quite different.
When we called \texttt{set{\_}up{\linebreak[0]}({\linebreak[0]}w{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}p{\linebreak[0]})}, the value of \texttt{p} (an empty
dictionary) was assigned to a new local variable \texttt{properties}.  Since the
value of \texttt{p} is an object reference, both variables now reference the same
memory location.  Modifying something inside \texttt{properties} will also
change \texttt{p}, just as if we had done the following sequence of assignments:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }p = {\{}{\}}
\pysrcprompt{>{}>{}> }properties = p
\pysrcprompt{>{}>{}> }properties{[}\pysrcstring{'pos'}{]} = \pysrcstring{'noun'}
\pysrcprompt{>{}>{}> }p
\pysrcoutput{{\{}'pos': 'noun'{\}}}}\end{alltt}

Thus, to understand Python's call-by-value parameter passing,
it is enough to understand Python's assignment operation.
We will address some closely related issues in our later discussion
of variable scope (\href{\#sec-variable-scope}{Section 9}).


%___________________________________________________________________________

\subsection{Functional Decomposition}
\label{functional-decomposition}

Well-structured programs usually make extensive use of functions.
When a block of program code grows longer than 10-20 lines, it is a
great help to readability if the code is broken up into one or more
functions, each one having a clear purpose.  This is analogous to
the way a good essay is divided into paragraphs, each expressing one main idea.

Functions provide an important kind of \emph{abstraction}.
They allow us to group multiple actions into a single, complex action,
and associate a name with it.
(Compare this with the way we combine the actions of
\docutilsroleexample{go} and \docutilsroleexample{bring back} into a single more complex action \docutilsroleexample{fetch}.)
When we use functions, the main program can be written at a higher level
of abstraction, making its structure transparent, e.g.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }data = load{\_}corpus()
\pysrcprompt{>{}>{}> }results = analyze(data)
\pysrcprompt{>{}>{}> }present(results)}\end{alltt}

Appropriate use of functions makes programs more readable and maintainable.
Additionally, it becomes possible to reimplement a function
— replacing the function's body with more efficient code —
without having to be concerned with the rest of the program.

Consider the \texttt{freq{\_}words} function in \href{\#freq-words1}{Listing 5.13}.
It updates the contents of a frequency distribution that is
passed in as a parameter, and it also prints a list of the
\docutilsrolemath{n} most frequent words.
\hypertarget{freq-words1}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{freq{\_}words}(url, freqdist, n):
    text = nltk.clean{\_}url(url)
    \pysrckeyword{for} word \pysrckeyword{in} nltk.wordpunct{\_}tokenize(text):
        freqdist.inc(word.lower())
    \pysrckeyword{print} freqdist.sorted(){[}:n{]}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }constitution = \pysrcstring{"http://www.archives.gov/national-archives-experience/charters/constitution{\_}transcript.html"}
\pysrcprompt{>{}>{}> }fd = nltk.FreqDist()
\pysrcprompt{>{}>{}> }freq{\_}words(constitution, fd, 20)
\pysrcoutput{{[}'the', 'of', 'charters', 'bill', 'constitution', 'rights', ',',}
\pysrcoutput{'declaration', 'impact', 'freedom', '-', 'making', 'independence'{]}}}\end{alltt}
\caption{}
\vspace{1ex}
\hrule
\end{pylisting}

This function has a number of problems.
The function has two side-effects: it modifies the contents of its second
parameter, and it prints a selection of the results it has computed.
The function would be easier to understand and to reuse elsewhere if we
initialize the \texttt{FreqDist{\linebreak[0]}({\linebreak[0]})} object inside the function (in the same place
it is populated), and if we moved the selection and display of results to the
calling program.  In \href{\#freq-words2}{Listing 5.14} we \index{refactor}\textbf{refactor} this function,
and simplify its interface by providing a single \texttt{url} parameter.
\hypertarget{freq-words2}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{freq{\_}words}(url):
    freqdist = nltk.FreqDist()
    text = nltk.clean{\_}url(url)
    \pysrckeyword{for} word \pysrckeyword{in} nltk.wordpunct{\_}tokenize(text):
        freqdist.inc(word.lower())
    return freqdist}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fd = freq{\_}words(constitution)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} fd.sorted(){[}:20{]}
\pysrcoutput{{[}'the', 'of', 'charters', 'bill', 'constitution', 'rights', ',',}
\pysrcoutput{'declaration', 'impact', 'freedom', '-', 'making', 'independence'{]}}}\end{alltt}
\caption{}
\vspace{1ex}
\hrule
\end{pylisting}

Note that we have now simplified the work of \texttt{freq{\_}words}
to the point that we can do its work with three lines of code:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }words = nltk.wordpunct{\_}tokenize(nltk.clean{\_}url(constitution))
\pysrcprompt{>{}>{}> }fd = nltk.FreqDist(word.lower() \pysrckeyword{for} word \pysrckeyword{in} words)
\pysrcprompt{>{}>{}> }fd.sorted(){[}:20{]}
\pysrcoutput{{[}'the', 'of', 'charters', 'bill', 'constitution', 'rights', ',',}
\pysrcoutput{'declaration', 'impact', 'freedom', '-', 'making', 'independence'{]}}}\end{alltt}


%___________________________________________________________________________

\subsection{Documentation (notes)}
\label{documentation-notes}
\begin{itemize}
\item {} 
some guidelines for literate programming (e.g. variable and function naming)

\item {} 
documenting functions (user-level and developer-level documentation)

\end{itemize}
\hypertarget{functions-as-arguments}{}

%___________________________________________________________________________

\subsection{Functions as Arguments}
\label{id41}
\label{functions-as-arguments}

So far the arguments we have passed into functions have been simple objects like
strings, or structured objects like lists.  These arguments allow us to parameterize
the behavior of a function.  As a result, functions are very flexible and powerful
abstractions, permitting us to repeatedly apply the \docutilsroleemphasis{same operation} on \docutilsroleemphasis{different data}.
Python also lets us pass a function as
an argument to another function.  Now we can abstract out the operation, and apply
a \docutilsroleemphasis{different operation} on the \docutilsroleemphasis{same data}.  As the following examples show,
we can pass the built-in function \texttt{len{\linebreak[0]}({\linebreak[0]})} or a user-defined function \texttt{last{\_}letter{\linebreak[0]}({\linebreak[0]})}
as parameters to another function:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{extract{\_}property}(prop):
\pysrcmore{... }    words = {[}\pysrcstring{'The'}, \pysrcstring{'dog'}, \pysrcstring{'gave'}, \pysrcstring{'John'}, \pysrcstring{'the'}, \pysrcstring{'newspaper'}{]}
\pysrcmore{... }    return {[}prop(word) \pysrckeyword{for} word \pysrckeyword{in} words{]}
\pysrcprompt{>{}>{}> }extract{\_}property(len)
\pysrcoutput{{[}3, 3, 4, 4, 3, 9{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{last{\_}letter}(word):
\pysrcmore{... }    return word{[}-1{]}
\pysrcprompt{>{}>{}> }extract{\_}property(last{\_}letter)
\pysrcoutput{{[}'e', 'g', 'e', 'n', 'e', 'r'{]}}}\end{alltt}

Surprisingly, \texttt{len} and \texttt{last{\_}letter} are objects that can be
passed around like lists and dictionaries.  Notice that parentheses
are only used after a function name if we are invoking the function;
when we are simply passing the function around as an object these are
not used.

Python provides us with one more way to define functions as arguments
to other functions, so-called \index{lambda expressions}\textbf{lambda expressions}.  Supposing there
was no need to use the above \texttt{last{\_}letter{\linebreak[0]}({\linebreak[0]})} function in multiple places,
and thus no need to give it a name.  We can equivalently write the following:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }extract{\_}property(\pysrckeyword{lambda} w: w{[}-1{]})
\pysrcoutput{{[}'e', 'g', 'e', 'n', 'e', 'r'{]}}}\end{alltt}

Our next example illustrates passing a function to the \texttt{sorted{\linebreak[0]}({\linebreak[0]})} function.
When we call the latter with a single argument (the list to be sorted),
it uses the built-in lexicographic comparison function \texttt{cmp{\linebreak[0]}({\linebreak[0]})}.
However, we can supply our own sort function, e.g. to sort by decreasing
length.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }words = \pysrcstring{'I turned off the spectroroute'}.split()
\pysrcprompt{>{}>{}> }sorted(words)
\pysrcoutput{{[}'I', 'off', 'spectroroute', 'the', 'turned'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }sorted(words, cmp)
\pysrcoutput{{[}'I', 'off', 'spectroroute', 'the', 'turned'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }sorted(words, \pysrckeyword{lambda} x, y: cmp(len(y), len(x)))
\pysrcoutput{{[}'spectroroute', 'turned', 'off', 'the', 'I'{]}}}\end{alltt}

In \href{\#sec-more-list-comprehensions}{5.2.5} we saw an example of filtering out
some items in a list comprehension, using an \texttt{\pysrckeyword{if}} test.
Similarly, we can restrict a list to just the lexical words, using
\texttt{{[}{\linebreak[0]}word{\linebreak[0]} \pysrckeyword{for} {\linebreak[0]}word{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}sent{\linebreak[0]} \pysrckeyword{if} {\linebreak[0]}is{\_}lexical{\linebreak[0]}({\linebreak[0]}word{\linebreak[0]}){\linebreak[0]}{]}}.  This is a little cumbersome
as it mentions the \texttt{word} variable three times.  A more compact way to express
the same thing is as follows.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{is{\_}lexical}(word):
\pysrcmore{... }    return word.lower() \pysrckeyword{not} \pysrckeyword{in} (\pysrcstring{'a'}, \pysrcstring{'an'}, \pysrcstring{'the'}, \pysrcstring{'that'}, \pysrcstring{'to'})
\pysrcprompt{>{}>{}> }sent = {[}\pysrcstring{'The'}, \pysrcstring{'dog'}, \pysrcstring{'gave'}, \pysrcstring{'John'}, \pysrcstring{'the'}, \pysrcstring{'newspaper'}{]}
\pysrcprompt{>{}>{}> }filter(is{\_}lexical, sent)
\pysrcoutput{{[}'dog', 'gave', 'John', 'newspaper'{]}}}\end{alltt}

{\noindent} The function \texttt{is{\_}lexical{\linebreak[0]}({\linebreak[0]}word{\linebreak[0]})} returns \texttt{True} just in case
\texttt{word}, when normalized to lowercase, is not in the given list.
This function is itself used as an argument to \texttt{filter{\linebreak[0]}({\linebreak[0]})}.
The \texttt{filter{\linebreak[0]}({\linebreak[0]})} function applies its first argument (a function) to each
item of its second (a sequence), only passing it through if the
function returns true for that item.  Thus \texttt{filter{\linebreak[0]}({\linebreak[0]}f{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}seq{\linebreak[0]})} is
equivalent to \texttt{{[}{\linebreak[0]}item{\linebreak[0]} \pysrckeyword{for} {\linebreak[0]}item{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}seq{\linebreak[0]} \pysrckeyword{if} {\linebreak[0]}apply{\linebreak[0]}({\linebreak[0]}f{\linebreak[0]},{\linebreak[0]}item{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}={\linebreak[0]}={\linebreak[0]} {\linebreak[0]}True{\linebreak[0]}{]}}.

Another helpful function, which like \texttt{filter{\linebreak[0]}({\linebreak[0]})} applies a function
to a sequence, is \texttt{map{\linebreak[0]}({\linebreak[0]})}.  Here is a simple
way to find the average length of a sentence in a section of the Brown Corpus:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }average(map(len, nltk.corpus.brown.sents(categories=\pysrcstring{'a'})))
\pysrcoutput{21.7508111616}}\end{alltt}

{\noindent}
Instead of \texttt{len{\linebreak[0]}({\linebreak[0]})}, we could have passed in any other function we liked:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = {[}\pysrcstring{'The'}, \pysrcstring{'dog'}, \pysrcstring{'gave'}, \pysrcstring{'John'}, \pysrcstring{'the'}, \pysrcstring{'newspaper'}{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{is{\_}vowel}(letter):
\pysrcmore{... }    return letter \pysrckeyword{in} \pysrcstring{"AEIOUaeiou"}
\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{vowelcount}(word):
\pysrcmore{... }    return len(filter(is{\_}vowel, word))
\pysrcprompt{>{}>{}> }map(vowelcount, sent)
\pysrcoutput{{[}1, 1, 2, 1, 1, 3{]}}}\end{alltt}

{\noindent}
Instead of using \texttt{filter{\linebreak[0]}({\linebreak[0]})} to call a named function \texttt{is{\_}vowel}, we can
define a lambda expression as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }map(\pysrckeyword{lambda} w: len(filter(\pysrckeyword{lambda} c: c \pysrckeyword{in} \pysrcstring{"AEIOUaeiou"}, w)), sent)
\pysrcoutput{{[}1, 1, 2, 1, 1, 3{]}}}\end{alltt}


%___________________________________________________________________________

\subsection{Exercises}
\label{id42}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Review the answers that you gave for the exercises in \href{\#sec-back-to-the-basics}{5.2},
and rewrite the code as one or more functions.

\item {} 
◑ In this section we saw examples of some special functions such as \texttt{filter{\linebreak[0]}({\linebreak[0]})} and
\texttt{map{\linebreak[0]}({\linebreak[0]})}.  Other functions in this family are \texttt{zip{\linebreak[0]}({\linebreak[0]})} and \texttt{reduce{\linebreak[0]}({\linebreak[0]})}.
Find out what these do, and write some code to try them out.
What uses might they have in language processing?

\item {} 
◑ Write a function that takes a list of words (containing duplicates) and
returns a list of words (with no duplicates) sorted by decreasing frequency.
E.g. if the input list contained 10 instances of the word \texttt{table} and 9 instances
of the word \texttt{chair}, then \texttt{table} would appear before \texttt{chair} in the output
list.

\item {} 
◑ Write a function that takes a text and a vocabulary as its arguments
and returns the set of words that appear in the text but not in the
vocabulary.  Both arguments can be represented as lists of strings.
Can you do this in a single line, using \texttt{set{\linebreak[0]}.{\linebreak[0]}difference{\linebreak[0]}({\linebreak[0]})}?

\item {} 
◑ As you saw, \texttt{zip{\linebreak[0]}({\linebreak[0]})} combines two lists into a single list
of pairs. What happens when the lists are of unequal lengths?
Define a function \texttt{myzip{\linebreak[0]}({\linebreak[0]})} that does something different with
unequal lists.

\item {} 
◑ Import the \texttt{itemgetter{\linebreak[0]}({\linebreak[0]})} function from the \texttt{operator} module in Python's
standard library (i.e. \texttt{\pysrckeyword{from} {\linebreak[0]}operator{\linebreak[0]} \pysrckeyword{import} {\linebreak[0]}itemgetter}).  Create a list
\texttt{words} containing several words.  Now try calling:
\texttt{sorted{\linebreak[0]}({\linebreak[0]}words{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}key{\linebreak[0]}={\linebreak[0]}itemgetter{\linebreak[0]}({\linebreak[0]}1{\linebreak[0]}){\linebreak[0]})}, and \texttt{sorted{\linebreak[0]}({\linebreak[0]}words{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}key{\linebreak[0]}={\linebreak[0]}itemgetter{\linebreak[0]}({\linebreak[0]}-{\linebreak[0]}1{\linebreak[0]}){\linebreak[0]})}.
Explain what \texttt{itemgetter{\linebreak[0]}({\linebreak[0]})} is doing.

\end{list}
\hypertarget{sec-algorithm-design-strategies}{}

%___________________________________________________________________________

\section{Algorithm Design Strategies}
\label{algorithm-design-strategies}
\label{sec-algorithm-design-strategies}

A major part of algorithmic problem solving is selecting or adapting
an appropriate algorithm for the problem at hand.  Whole books are written
on this topic (e.g. \cite{levitin2004}) and we only have space to introduce
some key concepts and elaborate on the approaches that are most prevalent
in natural language processing.

The best known strategy is known as \index{divide-and-conquer}\textbf{divide-and-conquer}.
We attack a problem of size \emph{n} by dividing it into two problems of size \emph{n/2},
solve these problems, and combine their results into a solution of the original problem.
\href{\#mergesort}{Figure 5.4} illustrates this approach for sorting a list of words.
\hypertarget{mergesort}{}\begin{figure}[htbp]
\centering

\scalebox{0.180000}{\includegraphics{../images/mergesort.png}}
\caption{Sorting by Divide-and-Conquer (Mergesort)}\end{figure}

Another strategy is \index{decrease-and-conquer}\textbf{decrease-and-conquer}.  In this approach, a small amount
of work on a problem of size \emph{n} permits us to reduce it to a problem of
size \emph{n/2}.  \href{\#binary-search}{Figure 5.5} illustrates this approach for the problem
of finding the index of an item in a sorted list.
\hypertarget{binary-search}{}\begin{figure}[htbp]
\centering

\scalebox{0.180000}{\includegraphics{../images/binary-search.png}}
\caption{Searching by Decrease-and-Conquer (Binary Search)}\end{figure}

A third well-known strategy is \index{transform-and-conquer}\textbf{transform-and-conquer}.  We attack a problem
by transforming it into an instance of a problem we already know how to solve.
For example, in order to detect duplicates entries in a list, we can \index{pre-sort}\textbf{pre-sort}
the list, then look for adjacent identical items, as shown in \href{\#presorting}{Listing 5.15}.
Our approach to n-gram chunking in \href{\#n-gram-chunking}{Section 6.5} is another case of
transform and conquer (why?).
\hypertarget{presorting}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{duplicates}(words):
    prev = None
    dup = {[}None{]}
    \pysrckeyword{for} word \pysrckeyword{in} sorted(words):
        \pysrckeyword{if} word == prev \pysrckeyword{and} word != dup{[}-1{]}:
            dup.append(word)
        \pysrckeyword{else}:
            prev = word
    return dup{[}1:{]}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }duplicates({[}\pysrcstring{'cat'}, \pysrcstring{'dog'}, \pysrcstring{'cat'}, \pysrcstring{'pig'}, \pysrcstring{'dog'}, \pysrcstring{'cat'}, \pysrcstring{'ant'}, \pysrcstring{'cat'}{]})
\pysrcoutput{{[}'cat', 'dog'{]}}}\end{alltt}
\caption{Presorting a list for duplicate detection}
\vspace{1ex}
\hrule
\end{pylisting}


%___________________________________________________________________________

\subsection{Recursion (notes)}
\label{recursion-notes}

We first saw recursion in \href{\#chap-words}{Chapter 2}, in a function that navigated
the hypernym hierarchy of WordNet...

Iterative solution:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{factorial}(n):
\pysrcmore{... }    result = 1
\pysrcmore{... }    \pysrckeyword{for} i \pysrckeyword{in} range(n):
\pysrcmore{... }        result *= (i+1)
\pysrcmore{... }    return result}\end{alltt}

Recursive solution (base case, induction step)
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{factorial}(n):
\pysrcmore{... }    \pysrckeyword{if} n == 1:
\pysrcmore{... }        return n
\pysrcmore{... }    \pysrckeyword{else}:
\pysrcmore{... }        return n * factorial(n-1)}\end{alltt}

{[}Simple example of recursion on strings.{]}

Generating all permutations of words, to check which ones are
grammatical:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{perms}(seq):
\pysrcmore{... }    \pysrckeyword{if} len(seq) <= 1:
\pysrcmore{... }        yield seq
\pysrcmore{... }    \pysrckeyword{else}:
\pysrcmore{... }        \pysrckeyword{for} perm \pysrckeyword{in} perms(seq{[}1:{]}):
\pysrcmore{... }            \pysrckeyword{for} i \pysrckeyword{in} range(len(perm)+1):
\pysrcmore{... }                yield perm{[}:i{]} + seq{[}0:1{]} + perm{[}i:{]}
\pysrcprompt{>{}>{}> }list(perms({[}\pysrcstring{'police'}, \pysrcstring{'fish'}, \pysrcstring{'cream'}{]}))
\pysrcoutput{{[}{[}'police', 'fish', 'cream'{]}, {[}'fish', 'police', 'cream'{]},}
\pysrcoutput{ {[}'fish', 'cream', 'police'{]}, {[}'police', 'cream', 'fish'{]},}
\pysrcoutput{ {[}'cream', 'police', 'fish'{]}, {[}'cream', 'fish', 'police'{]}{]}}}\end{alltt}


%___________________________________________________________________________

\subsection{Deeply Nested Objects (notes)}
\label{deeply-nested-objects-notes}

We can use recursive functions to build deeply-nested objects.
Building a letter trie, \href{\#trie}{Listing 5.16}.
\hypertarget{trie}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{insert}(trie, key, value):
    \pysrckeyword{if} key:
        first, rest = key{[}0{]}, key{[}1:{]}
        \pysrckeyword{if} first \pysrckeyword{not} \pysrckeyword{in} trie:
            trie{[}first{]} = {\{}{\}}
        insert(trie{[}first{]}, rest, value)
    \pysrckeyword{else}:
        trie{[}\pysrcstring{'value'}{]} = value}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }trie = {\{}{\}}
\pysrcprompt{>{}>{}> }insert(trie, \pysrcstring{'chat'}, \pysrcstring{'cat'})
\pysrcprompt{>{}>{}> }insert(trie, \pysrcstring{'chien'}, \pysrcstring{'dog'})
\pysrcprompt{>{}>{}> }trie{[}\pysrcstring{'c'}{]}{[}\pysrcstring{'h'}{]}
\pysrcoutput{{\{}'a': {\{}'t': {\{}'value': 'cat'{\}}{\}}, 'i': {\{}'e': {\{}'n': {\{}'value': 'dog'{\}}{\}}{\}}{\}}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }trie{[}\pysrcstring{'c'}{]}{[}\pysrcstring{'h'}{]}{[}\pysrcstring{'a'}{]}{[}\pysrcstring{'t'}{]}{[}\pysrcstring{'value'}{]}
\pysrcoutput{'cat'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }pprint.pprint(trie)
\pysrcoutput{{\{}'c': {\{}'h': {\{}'a': {\{}'t': {\{}'value': 'cat'{\}}{\}},}
\pysrcoutput{             'i': {\{}'e': {\{}'n': {\{}'value': 'dog'{\}}{\}}{\}}{\}}{\}}{\}}}}\end{alltt}
\caption{Building a Letter Trie}
\vspace{1ex}
\hrule
\end{pylisting}
\hypertarget{sec-dynamic-programming}{}

%___________________________________________________________________________

\subsection{Dynamic Programming}
\label{dynamic-programming}
\label{sec-dynamic-programming}

Dynamic programming is a general technique for designing algorithms
which is widely used in natural language processing.  The term
'programming' is used in a different sense to what you might expect,
to mean planning or scheduling.  Dynamic programming is used when a
problem contains overlapping sub-problems.  Instead of computing
solutions to these sub-problems repeatedly, we simply store them in a
lookup table.
In the remainder of this section we will introduce dynamic programming,
but in a rather different context to syntactic parsing.

Pingala was an Indian author who lived around the 5th century B.C.,
and wrote a treatise on Sanskrit prosody called the \emph{Chandas Shastra}.
Virahanka extended this work around the 6th century A.D., studying the
number of ways of combining short and long syllables to create a meter
of length \emph{n}.  He found, for example, that there are five ways to
construct a meter of length 4: \emph{V}$_{4}$ = \emph{{\{}LL, SSL, SLS,
LSS, SSSS{\}}}.  Observe that we can split \emph{V}$_{4}$ into two
subsets, those starting with \emph{L} and those starting with
\emph{S}, as shown in \href{\#v4}{(8)}.
\hypertarget{v4}{}\begin{itemize}
\item[(8)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\emph{V}$_{4}$~=~\\
~~LL,~LSS~\\
~~~~i.e.~L~prefixed~to~each~item~of~\emph{V}$_{2}$~=~{\{}L,~SS{\}}~\\
~~SSL,~SLS,~SSSS~\\
~~~~i.e.~S~prefixed~to~each~item~of~\emph{V}$_{3}$~=~{\{}SL,~LS,~SSS{\}}
}\end{quote}
\end{itemize}
\hypertarget{virahanka}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{virahanka1}(n):
    \pysrckeyword{if} n == 0:
        return {[}\pysrcstring{"{}"}{]}
    \pysrckeyword{elif} n == 1:
        return {[}\pysrcstring{"S"}{]}
    \pysrckeyword{else}:
        s = {[}\pysrcstring{"S"} + prosody \pysrckeyword{for} prosody \pysrckeyword{in} virahanka1(n-1){]}
        l = {[}\pysrcstring{"L"} + prosody \pysrckeyword{for} prosody \pysrckeyword{in} virahanka1(n-2){]}
        return s + l

\pysrckeyword{def} \pysrcdefname{virahanka2}(n):
    lookup = {[}{[}\pysrcstring{"{}"}{]}, {[}\pysrcstring{"S"}{]}{]}
    \pysrckeyword{for} i \pysrckeyword{in} range(n-1):
        s = {[}\pysrcstring{"S"} + prosody \pysrckeyword{for} prosody \pysrckeyword{in} lookup{[}i+1{]}{]}
        l = {[}\pysrcstring{"L"} + prosody \pysrckeyword{for} prosody \pysrckeyword{in} lookup{[}i{]}{]}
        lookup.append(s + l)
    return lookup{[}n{]}

\pysrckeyword{def} \pysrcdefname{virahanka3}(n, lookup={\{}0:{[}\pysrcstring{"{}"}{]}, 1:{[}\pysrcstring{"S"}{]}{\}}):
    \pysrckeyword{if} n \pysrckeyword{not} \pysrckeyword{in} lookup:
        s = {[}\pysrcstring{"S"} + prosody \pysrckeyword{for} prosody \pysrckeyword{in} virahanka3(n-1){]}
        l = {[}\pysrcstring{"L"} + prosody \pysrckeyword{for} prosody \pysrckeyword{in} virahanka3(n-2){]}
        lookup{[}n{]} = s + l
    return lookup{[}n{]}

\pysrckeyword{from} nltk \pysrckeyword{import} memoize
@memoize
\pysrckeyword{def} \pysrcdefname{virahanka4}(n):
    \pysrckeyword{if} n == 0:
        return {[}\pysrcstring{"{}"}{]}
    \pysrckeyword{elif} n == 1:
        return {[}\pysrcstring{"S"}{]}
    \pysrckeyword{else}:
        s = {[}\pysrcstring{"S"} + prosody \pysrckeyword{for} prosody \pysrckeyword{in} virahanka4(n-1){]}
        l = {[}\pysrcstring{"L"} + prosody \pysrckeyword{for} prosody \pysrckeyword{in} virahanka4(n-2){]}
        return s + l}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }virahanka1(4)
\pysrcoutput{{[}'SSSS', 'SSL', 'SLS', 'LSS', 'LL'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }virahanka2(4)
\pysrcoutput{{[}'SSSS', 'SSL', 'SLS', 'LSS', 'LL'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }virahanka3(4)
\pysrcoutput{{[}'SSSS', 'SSL', 'SLS', 'LSS', 'LL'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }virahanka4(4)
\pysrcoutput{{[}'SSSS', 'SSL', 'SLS', 'LSS', 'LL'{]}}}\end{alltt}
\caption{Three Ways to Compute Sanskrit Meter}
\vspace{1ex}
\hrule
\end{pylisting}

{\noindent}
With this observation, we can write a little recursive function called
\texttt{virahanka1{\linebreak[0]}({\linebreak[0]})} to compute these meters, shown in \href{\#virahanka}{Listing 5.17}.
Notice that, in order to compute \emph{V}$_{4}$ we first compute
\emph{V}$_{3}$ and \emph{V}$_{2}$.  But to compute \emph{V}$_{3}$,
we need to first compute \emph{V}$_{2}$ and \emph{V}$_{1}$.  This \index{call
structure}\textbf{call
structure} is depicted in \href{\#call-structure}{(9)}.
\hypertarget{call-structure}{}\begin{itemize}
\item[(9)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-2.pdf}}}\end{itemize}

{\noindent}
As you can see, \emph{V}$_{2}$ is computed twice.
This might not seem like a significant problem, but
it turns out to be rather wasteful as \emph{n} gets large:
to compute \emph{V}$_{20}$ using this recursive technique, we
would compute \emph{V}$_{2}$ 4,181 times;
and for \emph{V}$_{40}$ we would compute \emph{V}$_{2}$ 63,245,986 times!
A much better alternative is to store the value of \emph{V}$_{2}$ in a table
and look it up whenever we need it.  The same goes for other values, such
as \emph{V}$_{3}$ and so on.  Function \texttt{virahanka2{\linebreak[0]}({\linebreak[0]})} implements a
dynamic programming approach to the problem.  It works by filling up a
table (called \texttt{lookup}) with solutions to \emph{all} smaller instances of the
problem, stopping as soon as we reach the value we're interested in.
At this point we read off the value and return it.  Crucially, each
sub-problem is only ever solved once.

Notice that the approach taken in \texttt{virahanka2{\linebreak[0]}({\linebreak[0]})} is to solve smaller
problems on the way to solving larger problems.  Accordingly, this is known as the
\index{bottom-up}\textbf{bottom-up} approach to dynamic programming.  Unfortunately it turns out
to be quite wasteful for some applications, since it
may compute solutions to sub-problems that are never required for
solving the main problem.  This wasted computation can be avoided
using the \index{top-down}\textbf{top-down} approach to dynamic programming, which is
illustrated in the function \texttt{virahanka3{\linebreak[0]}({\linebreak[0]})} in \href{\#virahanka}{Listing 5.17}.
Unlike the bottom-up approach, this approach is recursive.  It avoids
the huge wastage of \texttt{virahanka1{\linebreak[0]}({\linebreak[0]})} by checking whether it has
previously stored the result.  If not, it computes the result
recursively and stores it in the table.  The last step is to return
the stored result.  The final method is to use a Python \index{decorator}\textbf{decorator}
called \texttt{memoize}, which takes care of the housekeeping work done
by \texttt{virahanka3{\linebreak[0]}({\linebreak[0]})} without cluttering up the program.

This concludes our brief introduction to dynamic programming.
We will encounter it again in \href{\#chap-advanced-parsing}{Chapter 8}.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Dynamic programming is a kind of \index{memoization}\textbf{memoization}.
A memoized function stores results of previous calls to the
function along with the supplied parameters.  If the function is
subsequently called with those parameters, it returns the
stored result instead of recalculating it.
\end{boxedminipage}
\end{sffamily}
\end{center}
\hypertarget{sec-timing}{}

%___________________________________________________________________________

\subsection{Timing (notes)}
\label{timing-notes}
\label{sec-timing}

We can easily test the efficiency gains made by the use of dynamic programming,
or any other putative performance enhancement, using the \texttt{timeit} module:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} timeit \pysrckeyword{import} Timer
\pysrcprompt{>{}>{}> }Timer(\pysrcstring{"PYTHON CODE"}, \pysrcstring{"INITIALIZATION CODE"}).timeit()}\end{alltt}

{[}MORE{]}


%___________________________________________________________________________

\subsection{Exercises}
\label{id44}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
◑ Write a recursive function \texttt{lookup{\linebreak[0]}({\linebreak[0]}trie{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}key{\linebreak[0]})} that looks up a key in a trie,
and returns the value it finds.  Extend the function to return a word when it is uniquely
determined by its prefix (e.g. \texttt{vanguard} is the only word that starts with \texttt{vang{\linebreak[0]}-},
so \texttt{lookup{\linebreak[0]}({\linebreak[0]}trie{\linebreak[0]},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}vang{\linebreak[0]}'})} should return the same thing as \texttt{lookup{\linebreak[0]}({\linebreak[0]}trie{\linebreak[0]},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}vanguard{\linebreak[0]}'})}).

\item {} 
◑ Read about string edit distance and the Levenshtein Algorithm.
Try the implementation provided in \texttt{nltk{\linebreak[0]}.{\linebreak[0]}edit{\_}dist{\linebreak[0]}({\linebreak[0]})}.
How is this using dynamic programming?  Does it use the bottom-up or
top-down approach?
{[}See also \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}norvig{\linebreak[0]}.{\linebreak[0]}com{\linebreak[0]}/{\linebreak[0]}spell{\linebreak[0]}-{\linebreak[0]}correct{\linebreak[0]}.{\linebreak[0]}html}{]}

\item {} 
◑ The Catalan numbers arise in many applications of combinatorial mathematics,
including the counting of parse trees (\href{\#chap-advanced-parsing}{Chapter 8}).  The series
can be defined as follows: C$_{0}$ = 1, and
C$_{n+1}$ = Σ$_{0..n}$ (C$_{i}$C$_{n-i}$).
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Write a recursive function to compute \docutilsrolemath{n}th Catalan number C$_{n}$

\item {} 
Now write another function that does this computation using dynamic programming

\item {} 
Use the \texttt{timeit} module to compare the performance of these functions as \docutilsrolemath{n}
increases.

\end{list}

\item {} 
★ Write a recursive function that pretty prints a trie in alphabetically
sorted order, as follows

chat: 'cat'
-{}-ien: 'dog'
-???: ???

\item {} 
★ Write a recursive function that processes text, locating the uniqueness point in
each word, and discarding the remainder of each word.  How much compression does this
give?  How readable is the resulting text?

\end{list}


%___________________________________________________________________________

\section{Conclusion}
\label{id45}

{[}TO DO{]}
% Revisit your solutions to exercises in earlier chapters, and try to improve
% them, applying your new understanding of structured programming. 


%___________________________________________________________________________

\section{Further Reading}
\label{id46}

\cite{harel2004}

\cite{levitin2004}

\href{http://docs.python.org/lib/typesseq-strings.html}{http://docs.python.org/lib/typesseq-strings.html}
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
% TODO: More Abney references 
% TODO: introduce trees more here, rather than in parsing chapter (e.g. t.node) 
\hypertarget{chap-chunk}{}\setcounter{chapter}{5}

%___________________________________________________________________________

\chapter{Partial Parsing and Interpretation}
\label{partial-parsing-and-interpretation}
\label{chap-chunk}


%___________________________________________________________________________

\section{Introduction}
\label{id49}

In processing natural language, we are looking for structure and meaning.
Two of the most common methods are
\index{segmentation}\textbf{segmentation} and \index{labeling}\textbf{labeling}.  Recall that in tokenization, we
\docutilsroleemphasis{segment} a sequence of characters into tokens, while in tagging we
\docutilsroleemphasis{label} each of these tokens.  Moreover, these two operations of
segmentation and labeling go hand in hand.  We break up a stream of
characters into linguistically meaningful segments (e.g., words) so
that we can classify those segments with their part-of-speech
categories.  The result of such classification is represented by
adding a label (e.g., part-of-speech tag) to the segment in question.

We will see that many tasks can be construed as a combination of
segmentation and labeling. However, this involves generalizing our
notion of segmentation to encompass \docutilsroleemphasis{sequences} of tokens. Suppose
that we are trying to recognize the names of people, locations and
organizations in a piece of text (a task that is usually called \index{Named
Entity Recognition}\textbf{Named
Entity Recognition}). Many of these names will involve more than
one token: \docutilsroleexample{Cecil H. Green}, \docutilsroleexample{Escondido Village}, \docutilsroleexample{Stanford
University}; indeed, some names may have sub-parts that are also
names: \docutilsroleexample{Cecil H. Green Library}, \docutilsroleexample{Escondido Village Conference
Service Center}. In Named Entity Recognition, therefore, we need
to be able to identify the beginning and end of multi-token
sequences.

Identifying the boundaries of specific types of word sequences is also
required when we want to recognize pieces of syntactic
structure. Suppose for example that as a preliminary to Named Entity
Recognition, we have decided that it would be useful to just pick out
noun phrases from a piece of text. To carry this out in a complete
way, we would probably want to use a proper syntactic parser. But
parsing can be quite challenging and computationally expensive —
is there an easier alternative? The answer is Yes: we can look for
sequences of part-of-speech tags in a tagged text, using one or more
patterns that capture the typical ingredients of a noun phrase.

For example, here is some Wall Street Journal text with noun phrases
marked using brackets:
\hypertarget{wsj-nx}{}\begin{itemize}
\item[(10)] 
{[} The/DT market/NN {]} for/IN {[} system-management/NN software/NN {]}
for/IN {[} Digital/NNP {]} {[} 's/POS hardware/NN {]} is/VBZ fragmented/JJ
enough/RB that/IN {[} a/DT giant/NN {]} such/JJ as/IN {[} Computer/NNP
Associates/NNPS {]} should/MD do/VB well/RB there/RB ./.
\end{itemize}

From the point of view of theoretical linguistics, we seem to have
been rather unorthodox in our use of the term ``noun phrase''; although
all the bracketed strings are noun phrases, not every noun phrase has
been captured. We will discuss this issue in more detail shortly. For
the moment, let's say that we are identifying noun ``chunks'' rather
than full noun phrases.

In chunking, we carry out segmentation and labeling of multi-token sequences,
as illustrated in \href{\#chunk-segmentation}{Figure 6.1}.  The smaller boxes show
word-level segmentation and labeling, while the large boxes show
higher-level segmentation and labeling.  It is these larger pieces
that we will call \index{chunks}\textbf{chunks}, and the process of identifying them is called
\index{chunking}\textbf{chunking}.
\hypertarget{chunk-segmentation}{}\begin{figure}[htbp]
\centering

\scalebox{0.154286}{\includegraphics{../images/chunk-segmentation.png}}
\caption{Segmentation and Labeling at both the Token and Chunk Levels}\end{figure}

Like tokenization, chunking can skip over material in the input.  Tokenization
omits white space and punctuation characters.  Chunking uses only a subset of
the tokens and leaves others out.

In this chapter, we will explore chunking in some depth, beginning
with the definition and representation of chunks.  We will see regular
expression and n-gram approaches to chunking, and will develop and
evaluate chunkers using the CoNLL-2000 chunking corpus.  Towards the
end of the chapter, we will look more briefly at Named Entity
Recognition and related tasks.
% Chunking is an efficient and robust method for identifying short
% phrases in text, or "chunks".  Chunks are `non-overlapping spans of
% text`:em:, usually consisting of a head word (such as a noun) and the
% adjacent modifiers and function words (such as adjectives and
% determiners).
% 
% There are other motivations for chunking: to locate information, and
% to ignore information.  In the former case, we may want to extract
% all noun phrases so they can be indexed.  A text retrieval system
% could use such an index to support efficient retrieval for queries
% involving terminological expressions.
% 
% The reverse side of the coin is to *ignore* information.  Suppose that
% we want to study syntactic patterns, finding particular verbs in a
% corpus and displaying their arguments.  For instance, here are some
% uses of the verb `gave`:lx: in the Wall Street Journal (in the Penn
% Treebank corpus sample).  After `np`:gc:\ -chunking, the internal
% details of each noun phrase have been suppressed, allowing us to see
% some higher-level patterns::
% 
%   gave NP
%   gave up NP in NP
%   gave NP up
%   gave NP NP
%   gave NP to NP
% 
% |nopar|
% In this way we can acquire information about the complementation
% patterns of a verb like `gave`:lx:, for use in the development of a
% grammar (see Chapter chap-parse_). 


%___________________________________________________________________________

\section{Defining and Representing Chunks}
\label{defining-and-representing-chunks}


%___________________________________________________________________________

\subsection{Chunking vs Parsing}
\label{chunking-vs-parsing}

Chunking is akin to parsing in the sense that it can be used to build
hierarchical structure over text.  There are several important
differences, however.  First, as noted above, chunking is not
exhaustive, and typically ignores some items in the surface string. In
fact, chunking is sometimes called \index{partial parsing}\textbf{partial parsing}. Second,
where parsing constructs nested structures that are arbitrarily deep,
chunking creates structures of fixed depth (typically depth 2).  These
chunks often correspond to the lowest level of grouping identified in
the full parse tree. This is illustrated in \href{\#parsing-chunking}{(11b)} below,
which shows an \docutilsrolecategory{np} chunk structure and a completely parsed
counterpart:
\hypertarget{parsing-chunking}{}\begin{itemize}
\item[(11)] \begin{itemize}
\item[a.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-3.pdf}}}\end{itemize}
\begin{itemize}
\item[b.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-4.pdf}}}\end{itemize}
\end{itemize}

A significant motivation for chunking is its robustness and efficiency
relative to parsing.  As we will see in \href{\#chap-parse}{Chapter 7}, parsing has problems with robustness,
given the difficulty in gaining broad coverage while minimizing
ambiguity.  Parsing is also relatively inefficient: the time taken to
parse a sentence grows with the cube of the length of the sentence,
while the time taken to chunk a sentence only grows linearly.


%___________________________________________________________________________

\subsection{Representing Chunks: Tags vs Trees}
\label{representing-chunks-tags-vs-trees}

As befits its intermediate status between tagging and parsing, chunk
structures can be represented using either tags or trees.  The most
widespread file representation uses so-called \index{IOB tags}\textbf{IOB tags}.  In this
scheme, each token is tagged with one of three special chunk tags,
\texttt{I} (inside), \texttt{O} (outside), or \texttt{B} (begin).  A token is tagged
as \texttt{B} if it marks the beginning of a chunk.  Subsequent tokens
within the chunk are tagged \texttt{I}.  All other tokens are tagged \texttt{O}.
The \texttt{B} and \texttt{I} tags are suffixed with the chunk type,
e.g. \texttt{B{\linebreak[0]}-{\linebreak[0]}NP}, \texttt{I{\linebreak[0]}-{\linebreak[0]}NP}.  Of course, it is not necessary to specify a
chunk type for tokens that appear outside a chunk, so these are just
labeled \texttt{O}. An example of this scheme is shown in \href{\#chunk-tagrep}{Figure 6.2}.
\hypertarget{chunk-tagrep}{}\begin{figure}[htbp]
\centering

\scalebox{0.154286}{\includegraphics{../images/chunk-tagrep.png}}
\caption{Tag Representation of Chunk Structures}\end{figure}

IOB tags have become the standard way to represent chunk structures in
files, and we will also be using this format.  Here is an example of
the file representation of the information in \href{\#chunk-tagrep}{Figure 6.2}:
\begin{quote}{\ttfamily \raggedright \noindent
\small
We~PRP~B-NP~\\
saw~VBD~O~\\
the~DT~B-NP~\\
little~JJ~I-NP~\\
yellow~JJ~I-NP~\\
dog~NN~I-NP
}\end{quote}
% Give example with a verb group? 

{\noindent} In this representation, there is one token per line, each with
its part-of-speech tag and its chunk tag.  We will see later that this
format permits us to represent more than one chunk type, so long as
the chunks do not overlap.

As we saw earlier, chunk structures can also be represented using
trees.  These have the benefit that each chunk is a constituent that
can be manipulated directly.  An example is shown in \href{\#chunk-treerep}{Figure 6.3}:
\hypertarget{chunk-treerep}{}\begin{figure}[htbp]
\centering

\scalebox{0.154286}{\includegraphics{../images/chunk-treerep.png}}
\caption{Tree Representation of Chunk Structures}\end{figure}

{\noindent}
NLTK uses trees for its internal representation of chunks, and
provides methods for reading and writing such trees to the IOB
format.  By now you should understand what chunks are, and how they
are represented.  In the next section, you will see how to build a
simple chunker.


%___________________________________________________________________________

\section{Chunking}
\label{chunking}

A \index{chunker}\textbf{chunker} finds contiguous, non-overlapping spans of related
tokens and groups them together into chunks.  Chunkers often operate
on tagged texts, and use the tags to make chunking decisions.  In this
section we will see how to write a special type of regular expression
over part-of-speech tags, and then how to combine these into a chunk
grammar.  Then we will set up a chunker to chunk some tagged text
according to the grammar.

Chunking in NLTK begins with tagged tokens.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tagged{\_}tokens = {[}(\pysrcstring{"the"}, \pysrcstring{"DT"}), (\pysrcstring{"little"}, \pysrcstring{"JJ"}), (\pysrcstring{"yellow"}, \pysrcstring{"JJ"}),
\pysrcmore{... }(\pysrcstring{"dog"}, \pysrcstring{"NN"}), (\pysrcstring{"barked"}, \pysrcstring{"VBD"}), (\pysrcstring{"at"}, \pysrcstring{"IN"}),  (\pysrcstring{"the"}, \pysrcstring{"DT"}), (\pysrcstring{"cat"}, \pysrcstring{"NN"}){]}}\end{alltt}

Next, we write regular expressions over tag sequences.  The
following example identifies noun phrases that consist of an
optional determiner, followed by any number of adjectives, then a
noun.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }cp = nltk.RegexpParser(\pysrcstring{"NP: {\{}<DT>?<JJ>*<NN>{\}}"})}\end{alltt}

{\noindent} We create a chunker \texttt{cp} that can then be used
repeatedly to parse tagged input.  The result of chunking is
a tree.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }cp.parse(tagged{\_}tokens).draw()}\end{alltt}

\scalebox{0.771429}{\includegraphics{tree_images/book-tree-5.pdf}}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Remember that our program samples assume you
begin your interactive session or your program with: \texttt{\pysrckeyword{import} {\linebreak[0]}nltk{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}re{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}pprint}
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\subsection{Tag Patterns}
\label{tag-patterns}

A \index{tag pattern}\textbf{tag pattern} is a sequence of part-of-speech tags delimited
using angle brackets, e.g. \texttt{<{\linebreak[0]}DT{\linebreak[0]}>{\linebreak[0]}<{\linebreak[0]}JJ{\linebreak[0]}>{\linebreak[0]}<{\linebreak[0]}NN{\linebreak[0]}>}.  Tag patterns are
the same as the regular expression patterns we have already seen,
except for two differences that make them easier to use for chunking.
First, angle brackets group their contents into atomic
units, so ``\texttt{<{\linebreak[0]}NN{\linebreak[0]}>{\linebreak[0]}+}'' matches one or more repetitions of the tag
\texttt{NN}; and ``\texttt{<{\linebreak[0]}NN{\linebreak[0]}|{\linebreak[0]}JJ{\linebreak[0]}>}'' matches the \texttt{NN} or \texttt{JJ}.  Second, the period wildcard operator is
constrained not to cross tag delimiters, so that ``\texttt{<{\linebreak[0]}N{\linebreak[0]}.{\linebreak[0]}*{\linebreak[0]}>}'' matches
any single tag starting with \texttt{N}, e.g. \texttt{NN}, \texttt{NNS}.

Now, consider the following noun phrases from the Wall Street
Journal:
\begin{quote}{\ttfamily \raggedright \noindent
\small
another/DT~sharp/JJ~dive/NN~\\
trade/NN~figures/NNS~\\
any/DT~new/JJ~policy/NN~measures/NNS~\\
earlier/JJR~stages/NNS~\\
Panamanian/JJ~dictator/NN~Manuel/NNP~Noriega/NNP
}\end{quote}
% TODO: how are they to view this CoNLL data? 

We can match these using a slight refinement of the first tag pattern
above: \texttt{<{\linebreak[0]}DT{\linebreak[0]}>{\linebreak[0]}?{\linebreak[0]}<{\linebreak[0]}JJ{\linebreak[0]}.{\linebreak[0]}*{\linebreak[0]}>{\linebreak[0]}*{\linebreak[0]}<{\linebreak[0]}NN{\linebreak[0]}.{\linebreak[0]}*{\linebreak[0]}>{\linebreak[0]}+}.  This can be used to chunk any sequence
of tokens beginning with an optional determiner \texttt{DT}, followed by
zero or more adjectives of any type \texttt{JJ{\linebreak[0]}.{\linebreak[0]}*} (including relative
adjectives like \texttt{earlier{\linebreak[0]}/{\linebreak[0]}JJR}), followed by one or more nouns of any
type \texttt{NN{\linebreak[0]}.{\linebreak[0]}*}.  It is easy to find many more difficult examples:
\begin{quote}{\ttfamily \raggedright \noindent
\small
his/PRP{\$}~Mansion/NNP~House/NNP~speech/NN~\\
the/DT~price/NN~cutting/VBG~\\
3/CD~{\%}/NN~to/TO~4/CD~{\%}/NN~\\
more/JJR~than/IN~10/CD~{\%}/NN~\\
the/DT~fastest/JJS~developing/VBG~trends/NNS~\\
's/POS~skill/NN
}\end{quote}

Your challenge will be to come up with tag patterns to cover these and
other examples.  A good way to learn about tag patterns is via a
graphical interface \texttt{nltk{\linebreak[0]}.{\linebreak[0]}draw{\linebreak[0]}.{\linebreak[0]}rechunkparser{\linebreak[0]}.{\linebreak[0]}demo{\linebreak[0]}({\linebreak[0]})}.


%___________________________________________________________________________

\subsection{Chunking with Regular Expressions}
\label{chunking-with-regular-expressions}

The chunker begins with a flat structure in which no tokens are
chunked.  Patterns are applied in turn, successively updating the
chunk structure.  Once all of the patterns have been applied, the
resulting chunk structure is returned.  \href{\#chunker1}{Listing 6.1} shows a
simple chunk grammar consisting of two patterns.  The first pattern
matches an optional determiner or possessive pronoun (recall that
\texttt{|} indicates disjunction), zero or more adjectives, then a
noun. The second rule matches one or more proper nouns.  We also
define some tagged tokens to be chunked, and run the chunker on this input.
\hypertarget{chunker1}{}
\begin{pylisting}
\begin{alltt}\small\textbf{grammar = r\pysrcstring{"{}"{}"}
\pysrcstring{  NP: {\{}<DT|PP{\textbackslash}{\$}>?<JJ>*<NN>{\}}   {\#} chunk determiner/possessive, adjectives and nouns}
\pysrcstring{      {\{}<NNP>+{\}}                {\#} chunk sequences of proper nouns}
\pysrcstring{"{}"{}"}
cp = nltk.RegexpParser(grammar)
tagged{\_}tokens = {[}(\pysrcstring{"Rapunzel"}, \pysrcstring{"NNP"}), (\pysrcstring{"let"}, \pysrcstring{"VBD"}), (\pysrcstring{"down"}, \pysrcstring{"RP"}), (\pysrcstring{"her"}, \pysrcstring{"PP{\$}"}), (\pysrcstring{"long"}, \pysrcstring{"JJ"}),
             (\pysrcstring{"golden"}, \pysrcstring{"JJ"}), (\pysrcstring{"hair"}, \pysrcstring{"NN"}){]}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} cp.parse(tagged{\_}tokens)
\pysrcoutput{(S}
\pysrcoutput{  (NP Rapunzel/NNP)}
\pysrcoutput{  let/VBD}
\pysrcoutput{  down/RP}
\pysrcoutput{  (NP her/PP{\$} long/JJ golden/JJ hair/NN))}}\end{alltt}
\caption{Simple Noun Phrase Chunker}
\vspace{1ex}
\hrule
\end{pylisting}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

The \texttt{{\$}} symbol is a special character in regular
expressions, and therefore needs to be escaped with the
backslash \texttt{{\textbackslash}} in order to match the tag \texttt{PP{\linebreak[0]}{\$}}.
\end{boxedminipage}
\end{sffamily}
\end{center}

If a tag pattern matches at overlapping locations, the first
match takes precedence.  For example, if we apply a rule that matches
two consecutive nouns to a text containing three consecutive nouns,
then only the first two nouns will be chunked:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nouns = {[}(\pysrcstring{"money"}, \pysrcstring{"NN"}), (\pysrcstring{"market"}, \pysrcstring{"NN"}), (\pysrcstring{"fund"}, \pysrcstring{"NN"}){]}
\pysrcprompt{>{}>{}> }grammar = \pysrcstring{"NP: {\{}<NN><NN>{\}}  {\#} Chunk two consecutive nouns"}
\pysrcprompt{>{}>{}> }cp = nltk.RegexpParser(grammar)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} cp.parse(nouns)
\pysrcoutput{(S (NP money/NN market/NN) fund/NN)}}\end{alltt}

Once we have created the chunk for \docutilsroleexample{money market}, we have
removed the context that would have permitted \docutilsroleexample{fund} to be
included in a chunk.  This issue would have been avoided with
a more permissive chunk rule, e.g. \texttt{NP{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}{\{}{\linebreak[0]}<{\linebreak[0]}NN{\linebreak[0]}>{\linebreak[0]}+{\linebreak[0]}{\}}}.


%___________________________________________________________________________

\subsection{Developing Chunkers}
\label{developing-chunkers}

Creating a good chunker usually requires several rounds of
development and testing, during which existing rules are refined and
new rules are added.  In order to diagnose any problems, it often
helps to trace the execution of a chunker, using its \texttt{trace} argument.
The tracing output shows the rules that are
applied, and uses braces to show the chunks that are created at each
stage of processing.
In \href{\#chunker2}{Listing 6.2}, two chunk patterns are applied to the input
sentence.  The first rule finds all sequences of three tokens whose
tags are \texttt{DT}, \texttt{JJ}, and \texttt{NN}, and the second rule finds any
sequence of tokens whose tags are either \texttt{DT} or \texttt{NN}.  We
set up two chunkers, one for each rule ordering, and test them on
the same input.
\hypertarget{chunker2}{}
\begin{pylisting}
\begin{alltt}\small\textbf{tagged{\_}tokens = {[}(\pysrcstring{"The"}, \pysrcstring{"DT"}), (\pysrcstring{"enchantress"}, \pysrcstring{"NN"}),
            (\pysrcstring{"clutched"}, \pysrcstring{"VBD"}), (\pysrcstring{"the"}, \pysrcstring{"DT"}), (\pysrcstring{"beautiful"}, \pysrcstring{"JJ"}), (\pysrcstring{"hair"}, \pysrcstring{"NN"}){]}
cp1 = nltk.RegexpParser(r\pysrcstring{"{}"{}"}
\pysrcstring{  NP: {\{}<DT><JJ><NN>{\}}      {\#} Chunk det+adj+noun}
\pysrcstring{      {\{}<DT|NN>+{\}}          {\#} Chunk sequences of NN and DT}
\pysrcstring{  "{}"{}"})
cp2 = nltk.RegexpParser(r\pysrcstring{"{}"{}"}
\pysrcstring{  NP: {\{}<DT|NN>+{\}}          {\#} Chunk sequences of NN and DT}
\pysrcstring{      {\{}<DT><JJ><NN>{\}}      {\#} Chunk det+adj+noun}
\pysrcstring{  "{}"{}"})}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} cp1.parse(tagged{\_}tokens, trace=1)
\pysrcoutput{{\#} Input:}
\pysrcoutput{ <DT>  <NN>  <VBD>  <DT>  <JJ>  <NN>}
\pysrcoutput{{\#} Chunk det+adj+noun:}
\pysrcoutput{ <DT>  <NN>  <VBD> {\{}<DT>  <JJ>  <NN>{\}}}
\pysrcoutput{{\#} Chunk sequences of NN and DT:}
\pysrcoutput{{\{}<DT>  <NN>{\}} <VBD> {\{}<DT>  <JJ>  <NN>{\}}}
\pysrcoutput{(S}
\pysrcoutput{  (NP The/DT enchantress/NN)}
\pysrcoutput{  clutched/VBD}
\pysrcoutput{  (NP the/DT beautiful/JJ hair/NN))}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{print} cp2.parse(tagged{\_}tokens, trace=1)
\pysrcoutput{{\#} Input:}
\pysrcoutput{ <DT>  <NN>  <VBD>  <DT>  <JJ>  <NN>}
\pysrcoutput{{\#} Chunk sequences of NN and DT:}
\pysrcoutput{{\{}<DT>  <NN>{\}} <VBD> {\{}<DT>{\}} <JJ> {\{}<NN>{\}}}
\pysrcoutput{{\#} Chunk det+adj+noun:}
\pysrcoutput{{\{}<DT>  <NN>{\}} <VBD> {\{}<DT>{\}} <JJ> {\{}<NN>{\}}}
\pysrcoutput{(S}
\pysrcoutput{  (NP The/DT enchantress/NN)}
\pysrcoutput{  clutched/VBD}
\pysrcoutput{  (NP the/DT)}
\pysrcoutput{  beautiful/JJ}
\pysrcoutput{  (NP hair/NN))}}\end{alltt}
\caption{Two Noun Phrase Chunkers Having Identical Rules in Different Orders}
\vspace{1ex}
\hrule
\end{pylisting}

Observe that when we chunk material that is already partially chunked,
the chunker will only create chunks that do not partially overlap
existing chunks.  In the case of \texttt{cp2}, the second rule
did not find any chunks, since all chunks that matched
its tag pattern overlapped with existing chunks.  As you can see,
you need to be careful to put chunk rules in the right order.

You may have noted that we have added explanatory comments, preceded
by \texttt{\pysrccomment{{\#}}}, to each of our tag rules. Although it is not strictly
necessary to do this, it's a helpful reminder of what a rule is meant
to do, and it is used as a header line for the output of a rule
application when tracing is on.

You might want to test out some of your rules on a corpus. One option
is to use the Brown corpus. However, you need to  remember that the
Brown tagset is different from the Penn Treebank tagset that we
have been using for our examples so far in this chapter; see \href{\#brown-tags}{Table 3.6} in \href{\#chap-tag}{Chapter 3} for a refresher. Because the Brown tagset
uses \texttt{NP} for proper nouns, in this example we have followed Abney
in labeling noun chunks as \texttt{NX}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }grammar = (r\pysrcstring{"{}"{}"}
\pysrcmore{... }\pysrcstring{   NX: {\{}<AT|AP|PP{\textbackslash}{\$}>?<JJ.*>?<NN.*>{\}}  {\#} Chunk article/numeral/possessive+adj+noun}
\pysrcmore{... }\pysrcstring{       {\{}<NP>+{\}}                       {\#} Chunk one or more proper nouns}
\pysrcmore{... }\pysrcstring{"{}"{}"})
\pysrcprompt{>{}>{}> }cp = nltk.RegexpParser(grammar)
\pysrcprompt{>{}>{}> }sent = nltk.corpus.brown.tagged{\_}sents(categories=\pysrcstring{'a'}){[}112{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{print} cp.parse(sent)
\pysrcoutput{(S}
\pysrcoutput{  (NX His/PP{\$} contention/NN)}
\pysrcoutput{  was/BEDZ}
\pysrcoutput{  denied/VBN}
\pysrcoutput{  by/IN}
\pysrcoutput{  (NX several/AP bankers/NNS)}
\pysrcoutput{  ,/,}
\pysrcoutput{  including/IN}
\pysrcoutput{  (NX Scott/NP Hudson/NP)}
\pysrcoutput{  of/IN}
\pysrcoutput{  (NX Sherman/NP)}
\pysrcoutput{  ,/,}
\pysrcoutput{  (NX Gaynor/NP B./NP Jones/NP)}
\pysrcoutput{  of/IN}
\pysrcoutput{  (NX Houston/NP)}
\pysrcoutput{  ,/,}
\pysrcoutput{  (NX J./NP B./NP Brady/NP)}
\pysrcoutput{  of/IN}
\pysrcoutput{  (NX Harlingen/NP)}
\pysrcoutput{  and/CC}
\pysrcoutput{  (NX Howard/NP Cox/NP)}
\pysrcoutput{  of/IN}
\pysrcoutput{  (NX Austin/NP)}
\pysrcoutput{  ./.)}}\end{alltt}


%___________________________________________________________________________

\subsection{Exercises}
\label{id50}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ \textbf{Chunk Grammar Development:}
Try developing a series of chunking rules using the graphical
interface accessible via \texttt{nltk{\linebreak[0]}.{\linebreak[0]}draw{\linebreak[0]}.{\linebreak[0]}rechunkparser{\linebreak[0]}.{\linebreak[0]}demo{\linebreak[0]}({\linebreak[0]})}

\item {} 
☼ \textbf{Chunking Demonstration:}
Run the chunking demonstration: \texttt{nltk{\linebreak[0]}.{\linebreak[0]}chunk{\linebreak[0]}.{\linebreak[0]}demo{\linebreak[0]}({\linebreak[0]})}

\item {} 
☼ \textbf{IOB Tags:}
The IOB format categorizes tagged tokens as \texttt{I},
\texttt{O} and \texttt{B}.  Why are three tags necessary?  What
problem would be caused if we used \texttt{I} and \texttt{O} tags
exclusively?

\item {} 
☼ Write a tag pattern to match noun phrases containing plural head nouns,
e.g. ``many/JJ researchers/NNS'', ``two/CD weeks/NNS'', ``both/DT new/JJ positions/NNS''.
Try to do this by generalizing the tag pattern that handled singular
noun phrases.

\item {} 
◑ Write a tag pattern to cover noun phrases that contain gerunds,
e.g. ``the/DT receiving/VBG end/NN'', ``assistant/NN managing/VBG editor/NN''.
Add these patterns to the grammar, one per line.  Test your work using
some tagged sentences of your own devising.

\item {} 
◑ Write one or more tag patterns to handle coordinated noun phrases,
e.g. ``July/NNP and/CC August/NNP'',
``all/DT your/PRP{\$} managers/NNS and/CC supervisors/NNS'',
``company/NN courts/NNS and/CC adjudicators/NNS''.

\end{list}


%___________________________________________________________________________

\section{Scaling Up}
\label{id51}

Now you have a taste of what chunking can do, but we have not
explained how to carry out a quantitative evaluation of chunkers.  For
this, we need to get access to a corpus that has been annotated not
only with parts-of-speech, but also with chunk information.  We will
begin by looking at the mechanics of converting IOB format into an
NLTK tree, then at how this is done on a larger scale using a
chunked corpus directly.  We will see how to use the corpus to score
the accuracy of a chunker, then look some more flexible ways to
manipulate chunks.  Our focus throughout will be on scaling up the
coverage of a chunker.


%___________________________________________________________________________

\subsection{Reading IOB Format and the CoNLL 2000 Corpus}
\label{reading-iob-format-and-the-conll-2000-corpus}

Using the \texttt{corpora} module we can load Wall Street Journal
text that has been tagged, then chunked using the IOB notation.  The
chunk categories provided in this corpus are \docutilsrolecategory{np}, \docutilsrolecategory{vp} and \docutilsrolecategory{pp}.  As we
have seen, each sentence is represented using multiple lines, as shown
below:
\begin{quote}{\ttfamily \raggedright \noindent
\small
he~PRP~B-NP~\\
accepted~VBD~B-VP~\\
the~DT~B-NP~\\
position~NN~I-NP~\\
...
}\end{quote}

{\noindent} A conversion function \texttt{chunk{\linebreak[0]}.{\linebreak[0]}conllstr2tree{\linebreak[0]}({\linebreak[0]})} builds a tree
representation from one of these multi-line strings.  Moreover, it
permits us to choose any subset of the three chunk types to use.  The
example below produces only \docutilsrolecategory{np} chunks:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text = \pysrcstring{'{}'{}'}
\pysrcmore{... }\pysrcstring{he PRP B-NP}
\pysrcmore{... }\pysrcstring{accepted VBD B-VP}
\pysrcmore{... }\pysrcstring{the DT B-NP}
\pysrcmore{... }\pysrcstring{position NN I-NP}
\pysrcmore{... }\pysrcstring{of IN B-PP}
\pysrcmore{... }\pysrcstring{vice NN B-NP}
\pysrcmore{... }\pysrcstring{chairman NN I-NP}
\pysrcmore{... }\pysrcstring{of IN B-PP}
\pysrcmore{... }\pysrcstring{Carlyle NNP B-NP}
\pysrcmore{... }\pysrcstring{Group NNP I-NP}
\pysrcmore{... }\pysrcstring{, , O}
\pysrcmore{... }\pysrcstring{a DT B-NP}
\pysrcmore{... }\pysrcstring{merchant NN I-NP}
\pysrcmore{... }\pysrcstring{banking NN I-NP}
\pysrcmore{... }\pysrcstring{concern NN I-NP}
\pysrcmore{... }\pysrcstring{. . O}
\pysrcmore{... }\pysrcstring{'{}'{}'}
\pysrcprompt{>{}>{}> }nltk.chunk.conllstr2tree(text, chunk{\_}types=(\pysrcstring{'NP'},)).draw()}\end{alltt}

\scalebox{0.617143}{\includegraphics{tree_images/book-tree-6.pdf}}

We can use the NLTK corpus module to access a larger amount of chunked
text.  The CoNLL 2000 corpus contains 270k words of Wall Street
Journal text, divided into ``train'' and ``test'' portions, annotated with
part-of-speech tags and chunk tags in the IOB format.  We can access
the data using an NLTK corpus reader called \texttt{conll2000}.  Here is an
example that reads the 100th sentence of the ``train'' portion of the corpus:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.corpus.conll2000.chunked{\_}sents(\pysrcstring{'train.txt'}){[}99{]}
\pysrcoutput{(S}
\pysrcoutput{  (PP Over/IN)}
\pysrcoutput{  (NP a/DT cup/NN)}
\pysrcoutput{  (PP of/IN)}
\pysrcoutput{  (NP coffee/NN)}
\pysrcoutput{  ,/,}
\pysrcoutput{  (NP Mr./NNP Stone/NNP)}
\pysrcoutput{  (VP told/VBD)}
\pysrcoutput{  (NP his/PRP{\$} story/NN)}
\pysrcoutput{  ./.)}}\end{alltt}

{\noindent}
This  showed three chunk types, for \docutilsrolecategory{np}, \docutilsrolecategory{vp} and \docutilsrolecategory{pp}.
We can also select which chunk types to read:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.corpus.conll2000.chunked{\_}sents(\pysrcstring{'train.txt'}, chunk{\_}types=(\pysrcstring{'NP'},)){[}99{]}
\pysrcoutput{(S}
\pysrcoutput{  Over/IN}
\pysrcoutput{  (NP a/DT cup/NN)}
\pysrcoutput{  of/IN}
\pysrcoutput{  (NP coffee/NN)}
\pysrcoutput{  ,/,}
\pysrcoutput{  (NP Mr./NNP Stone/NNP)}
\pysrcoutput{  told/VBD}
\pysrcoutput{  (NP his/PRP{\$} story/NN)}
\pysrcoutput{  ./.)}}\end{alltt}


%___________________________________________________________________________

\subsection{Simple Evaluation and Baselines}
\label{simple-evaluation-and-baselines}

Armed with a corpus, it is now possible to carry out some simple evaluation.
We start off by establishing a baseline for the trivial chunk parser
\texttt{cp} that creates no chunks:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }cp = nltk.RegexpParser(\pysrcstring{"{}"})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.chunk.accuracy(cp, nltk.corpus.conll2000.chunked{\_}sents(\pysrcstring{'train.txt'}, chunk{\_}types=(\pysrcstring{'NP'},)))
\pysrcoutput{0.440845995079}}\end{alltt}

This indicates that more than a third of the words are tagged with
\texttt{O} (i.e., not in an \docutilsrolecategory{np} chunk).  Now let's try a naive regular
expression chunker that looks for tags (e.g., \texttt{CD}, \texttt{DT}, \texttt{JJ},
etc.) beginning with letters that are typical of noun phrase tags:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }grammar = r\pysrcstring{"NP: {\{}<{[}CDJNP{]}.*>+{\}}"}
\pysrcprompt{>{}>{}> }cp = nltk.RegexpParser(grammar)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.chunk.accuracy(cp, nltk.corpus.conll2000.chunked{\_}sents(\pysrcstring{'train.txt'}, chunk{\_}types=(\pysrcstring{'NP'},)))
\pysrcoutput{0.874479872666}}\end{alltt}

As you can see, this approach achieves pretty good results. In
order to develop a more data-driven approach, let's define a function
\texttt{chunked{\_}tags{\linebreak[0]}({\linebreak[0]})} that takes some chunked data
and sets up a conditional frequency distribution.
For each tag, it counts up the number of times the tag
occurs inside an \docutilsrolecategory{np} chunk (the \texttt{True} case, where \texttt{chtag} is
\texttt{B{\linebreak[0]}-{\linebreak[0]}NP} or \texttt{I{\linebreak[0]}-{\linebreak[0]}NP}), or outside a chunk (the \texttt{False} case, where
\texttt{chtag} is \texttt{O}).  It returns a list of those tags that occur
inside chunks more often than outside chunks.
\hypertarget{chunker3}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{chunked{\_}tags}(train):
    \pysrcstring{"{}"{}"Generate a list of tags that tend to appear inside chunks"{}"{}"}
    cfdist = nltk.ConditionalFreqDist()
    \pysrckeyword{for} t \pysrckeyword{in} train:
        \pysrckeyword{for} word, tag, chtag \pysrckeyword{in} nltk.chunk.tree2conlltags(t):
            \pysrckeyword{if} chtag == \pysrcstring{"O"}:
                cfdist{[}tag{]}.inc(False)
            \pysrckeyword{else}:
                cfdist{[}tag{]}.inc(True)
    return {[}tag \pysrckeyword{for} tag \pysrckeyword{in} cfdist.conditions() \pysrckeyword{if} cfdist{[}tag{]}.max() == True{]}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }train{\_}sents = nltk.corpus.conll2000.chunked{\_}sents(\pysrcstring{'train.txt'}, chunk{\_}types=(\pysrcstring{'NP'},))
\pysrcprompt{>{}>{}> }\pysrckeyword{print} chunked{\_}tags(train{\_}sents)
\pysrcoutput{{[}'PRP{\$}', 'WDT', 'JJ', 'WP', 'DT', '{\#}', '{\$}', 'NN', 'FW', 'POS',}
\pysrcoutput{'PRP', 'NNS', 'NNP', 'PDT', 'RBS', 'EX', 'WP{\$}', 'CD', 'NNPS', 'JJS', 'JJR'{]}}}\end{alltt}
\caption{Capturing the conditional frequency of NP Chunk Tags}
\vspace{1ex}
\hrule
\end{pylisting}

The next step is to convert this list of tags into a tag pattern.  To
do this we need to ``escape'' all non-word characters, by preceding them
with a backslash.  Then we need to join them into a disjunction.  This
process would convert a tag list \texttt{{[}\pysrcstring{'{\linebreak[0]}NN{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}NN{\linebreak[0]}{\textbackslash}{\linebreak[0]}{\$}{\linebreak[0]}'}{]}} into the tag pattern
\texttt{<{\linebreak[0]}NN{\linebreak[0]}|{\linebreak[0]}NN{\linebreak[0]}{\textbackslash}{\linebreak[0]}{\$}{\linebreak[0]}>}.  The following function does this work, and returns a
regular expression chunker:
\hypertarget{chunker4}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{baseline{\_}chunker}(train):
    chunk{\_}tags = {[}re.sub(r\pysrcstring{'({\textbackslash}W)'}, r\pysrcstring{'{\textbackslash}{\textbackslash}{\textbackslash}1'}, tag)
                  \pysrckeyword{for} tag \pysrckeyword{in} chunked{\_}tags(train){]}
    grammar = \pysrcstring{'NP: {\{}<{\%}s>+{\}}'} {\%} \pysrcstring{'|'}.join(chunk{\_}tags)
    return nltk.RegexpParser(grammar)}\end{alltt}
\caption{Deriving a Regexp Chunker from Training Data}
\vspace{1ex}
\hrule
\end{pylisting}

{\noindent}
The final step is to train this chunker and test its accuracy (this
time on the ``test'' portion of the corpus, i.e., data not seen during training):
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }train{\_}sents = nltk.corpus.conll2000.chunked{\_}sents(\pysrcstring{'train.txt'}, chunk{\_}types=(\pysrcstring{'NP'},))
\pysrcprompt{>{}>{}> }test{\_}sents  = nltk.corpus.conll2000.chunked{\_}sents(\pysrcstring{'test.txt'}, chunk{\_}types=(\pysrcstring{'NP'},))
\pysrcprompt{>{}>{}> }cp = baseline{\_}chunker(train{\_}sents)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.chunk.accuracy(cp, test{\_}sents)
\pysrcoutput{0.914262194736}}\end{alltt}


%___________________________________________________________________________

\subsection{Splitting and Merging (incomplete)}
\label{splitting-and-merging-incomplete}

{[}Notes: the above approach creates chunks that are too large,
e.g. \docutilsroleexample{the cat the dog chased} would be given a single \docutilsrolecategory{np} chunk
because it does not detect that determiners introduce new chunks.
For this we would need a rule to split an \docutilsrolecategory{np} chunk
prior to any determiner, using a pattern like:
\texttt{\pysrcstring{"{\linebreak[0]}NP{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}<{\linebreak[0]}.{\linebreak[0]}*{\linebreak[0]}>{\linebreak[0]}{\}}{\linebreak[0]}{\{}{\linebreak[0]}<{\linebreak[0]}DT{\linebreak[0]}>{\linebreak[0]}"}}.  We can also merge chunks, e.g.
\texttt{\pysrcstring{"{\linebreak[0]}NP{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}<{\linebreak[0]}NN{\linebreak[0]}>{\linebreak[0]}{\{}{\linebreak[0]}{\}}{\linebreak[0]}<{\linebreak[0]}NN{\linebreak[0]}>{\linebreak[0]}"}}.{]}


%___________________________________________________________________________

\subsection{Chinking}
\label{chinking}

Sometimes it is easier to define what we \emph{don't} want to include in a
chunk than it is to define what we \emph{do} want to include.  In these
cases, it may be easier to build a chunker using a method called
\index{chinking}\textbf{chinking}.

Following \cite{abney1996pst}, we define a \index{chink}\textbf{chink} as a sequence
of tokens that is not included in a chunk.
In the following example, \texttt{barked{\linebreak[0]}/{\linebreak[0]}VBD{\linebreak[0]} {\linebreak[0]}at{\linebreak[0]}/{\linebreak[0]}IN} is a chink:
\begin{quote}{\ttfamily \raggedright \noindent
\small
{[}~the/DT~little/JJ~yellow/JJ~dog/NN~{]}~barked/VBD~at/IN~{[}~the/DT~cat/NN~{]}
}\end{quote}

Chinking is the process of removing a sequence of tokens from a
chunk.  If the sequence of tokens spans an entire chunk, then the
whole chunk is removed; if the sequence of tokens appears in the
middle of the chunk, these tokens are removed, leaving two chunks
where there was only one before.  If the sequence is at the beginning
or end of the chunk, these tokens are removed, and a smaller chunk
remains.  These three possibilities are illustrated in \href{\#id53}{Table 6.1}.
\hypertarget{id53}{}
\leavevmode
\begin{longtable}[c]{|p{0.168\locallinewidth}|p{0.261\locallinewidth}|p{0.238\locallinewidth}|p{0.226\locallinewidth}|}
\hline
 & 
Entire chunk
 & 
Middle of a chunk
 & 
End of a chunk
 \\
\hline

\emph{Input}
 & 
{[}a/DT little/JJ
dog/NN{]}
 & 
{[}a/DT little/JJ
dog/NN{]}
 & 
{[}a/DT little/JJ
dog/NN{]}
 \\
\hline

\emph{Operation}
 & 
Chink ``DT JJ NN''
 & 
Chink ``JJ''
 & 
Chink ``NN''
 \\
\hline

\emph{Pattern}
 & 
``{\}}DT JJ NN{\{}''
 & 
``{\}}JJ{\{}''
 & 
``{\}}NN{\{}''
 \\
\hline

\emph{Output}
 & 
a/DT little/JJ
dog/NN
 & 
{[}a/DT{]} little/JJ
{[}dog/NN{]}
 & 
{[}a/DT little/JJ{]}
dog/NN
 \\
\hline
\caption{
Three chinking rules applied to the same chunk
}\end{longtable}

In the following grammar, we put the entire sentence into a single
chunk, then excise the chink:
\hypertarget{chinker}{}
\begin{pylisting}
\begin{alltt}\small\textbf{grammar = r\pysrcstring{"{}"{}"}
\pysrcstring{  NP:}
\pysrcstring{    {\{}<.*>+{\}}          {\#} Chunk everything}
\pysrcstring{    {\}}<VBD|IN>+{\{}      {\#} Chink sequences of VBD and IN}
\pysrcstring{  "{}"{}"}
tagged{\_}tokens = {[}(\pysrcstring{"the"}, \pysrcstring{"DT"}), (\pysrcstring{"little"}, \pysrcstring{"JJ"}), (\pysrcstring{"yellow"}, \pysrcstring{"JJ"}),
       (\pysrcstring{"dog"}, \pysrcstring{"NN"}), (\pysrcstring{"barked"}, \pysrcstring{"VBD"}), (\pysrcstring{"at"}, \pysrcstring{"IN"}),  (\pysrcstring{"the"}, \pysrcstring{"DT"}), (\pysrcstring{"cat"}, \pysrcstring{"NN"}){]}
cp = nltk.RegexpParser(grammar)}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} cp.parse(tagged{\_}tokens)
\pysrcoutput{(S}
\pysrcoutput{  (NP the/DT little/JJ yellow/JJ dog/NN)}
\pysrcoutput{  barked/VBD}
\pysrcoutput{  at/IN}
\pysrcoutput{  (NP the/DT cat/NN))}
\pysrcoutput{}\pysrcprompt{>{}>{}> }test{\_}sents = nltk.corpus.conll2000.chunked{\_}sents(\pysrcstring{'test.txt'}, chunk{\_}types=(\pysrcstring{'NP'},))
\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.chunk.accuracy(cp, test{\_}sents)
\pysrcoutput{0.581041433607}}\end{alltt}
\caption{Simple Chinker}
\vspace{1ex}
\hrule
\end{pylisting}

{\noindent}
A chunk grammar can use any number of chunking and chinking patterns
in any order.


%___________________________________________________________________________

\subsection{Multiple Chunk Types (incomplete)}
\label{multiple-chunk-types-incomplete}

So far we have only developed \docutilsrolecategory{np} chunkers.  However, as we saw earlier
in the chapter, the CoNLL chunking data is also annotated for \docutilsrolecategory{pp} and
\docutilsrolecategory{vp} chunks.  Here is an example, to show the structure we get from the
corpus and the flattened version that will be used as input to the parser.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }example = nltk.corpus.conll2000.chunked{\_}sents(\pysrcstring{'train.txt'}){[}99{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{print} example
\pysrcoutput{(S}
\pysrcoutput{  (PP Over/IN)}
\pysrcoutput{  (NP a/DT cup/NN)}
\pysrcoutput{  (PP of/IN)}
\pysrcoutput{  (NP coffee/NN)}
\pysrcoutput{  ,/,}
\pysrcoutput{  (NP Mr./NNP Stone/NNP)}
\pysrcoutput{  (VP told/VBD)}
\pysrcoutput{  (NP his/PRP{\$} story/NN)}
\pysrcoutput{  ./.)}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{print} example.flatten()
\pysrcoutput{(S}
\pysrcoutput{  Over/IN}
\pysrcoutput{  a/DT}
\pysrcoutput{  cup/NN}
\pysrcoutput{  of/IN}
\pysrcoutput{  coffee/NN}
\pysrcoutput{  ,/,}
\pysrcoutput{  Mr./NNP}
\pysrcoutput{  Stone/NNP}
\pysrcoutput{  told/VBD}
\pysrcoutput{  his/PRP{\$}}
\pysrcoutput{  story/NN}
\pysrcoutput{  ./.)}}\end{alltt}

Now we can set up a multi-stage chunk grammar, as shown in
\href{\#multistage-chunker}{Listing 6.6}.  It has a stage for each of the chunk types.
\hypertarget{multistage-chunker}{}
\begin{pylisting}
\begin{alltt}\small\textbf{cp = nltk.RegexpParser(r\pysrcstring{"{}"{}"}
\pysrcstring{  NP: {\{}<DT>?<JJ>*<NN.*>+{\}}    {\#} noun phrase chunks}
\pysrcstring{  VP: {\{}<TO>?<VB.*>{\}}          {\#} verb phrase chunks}
\pysrcstring{  PP: {\{}<IN>{\}}                 {\#} prepositional phrase chunks}
\pysrcstring{  "{}"{}"})}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }example = nltk.corpus.conll2000.chunked{\_}sents(\pysrcstring{'train.txt'}){[}99{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{print} cp.parse(example.flatten(), trace=1)
\pysrcoutput{{\#} Input:}
\pysrcoutput{ <IN>  <DT>  <NN>  <IN>  <NN>  <,>  <NNP>  <NNP>  <VBD>  <PRP{\$}>  <NN>  <.>}
\pysrcoutput{{\#} noun phrase chunks:}
\pysrcoutput{ <IN> {\{}<DT>  <NN>{\}} <IN> {\{}<NN>{\}} <,> {\{}<NNP>  <NNP>{\}} <VBD>  <PRP{\$}> {\{}<NN>{\}} <.>}
\pysrcoutput{{\#} Input:}
\pysrcoutput{ <IN>  <NP>  <IN>  <NP>  <,>  <NP>  <VBD>  <PRP{\$}>  <NP>  <.>}
\pysrcoutput{{\#} verb phrase chunks:}
\pysrcoutput{ <IN>  <NP>  <IN>  <NP>  <,>  <NP> {\{}<VBD>{\}} <PRP{\$}>  <NP>  <.>}
\pysrcoutput{{\#} Input:}
\pysrcoutput{ <IN>  <NP>  <IN>  <NP>  <,>  <NP>  <VP>  <PRP{\$}>  <NP>  <.>}
\pysrcoutput{{\#} prepositional phrase chunks:}
\pysrcoutput{{\{}<IN>{\}} <NP> {\{}<IN>{\}} <NP>  <,>  <NP>  <VP>  <PRP{\$}>  <NP>  <.>}
\pysrcoutput{(S}
\pysrcoutput{  (PP Over/IN)}
\pysrcoutput{  (NP a/DT cup/NN)}
\pysrcoutput{  (PP of/IN)}
\pysrcoutput{  (NP coffee/NN)}
\pysrcoutput{  ,/,}
\pysrcoutput{  (NP Mr./NNP Stone/NNP)}
\pysrcoutput{  (VP told/VBD)}
\pysrcoutput{  his/PRP{\$}}
\pysrcoutput{  (NP story/NN)}
\pysrcoutput{  ./.)}}\end{alltt}
\caption{A Multistage Chunker}
\vspace{1ex}
\hrule
\end{pylisting}


%___________________________________________________________________________

\subsection{Evaluating Chunk Parsers}
\label{evaluating-chunk-parsers}

An easy way to evaluate a chunk parser is to take some already chunked
text, strip off the chunks, rechunk it, and compare the result with
the original chunked text.  The \texttt{ChunkScore{\linebreak[0]}.{\linebreak[0]}score{\linebreak[0]}({\linebreak[0]})} function takes
the correctly chunked sentence as its first argument, and the newly
chunked version as its second argument, and compares them.  It reports
the fraction of actual chunks that were found (recall), the fraction
of hypothesized chunks that were correct (precision), and a combined
score, the F-measure (the harmonic mean of precision and recall).

A number of different metrics can be used to evaluate chunk parsers.
We will concentrate on a class of metrics that can be derived from two
sets:
\begin{itemize}
\item {} 
\textbf{guessed}: The set of chunks returned by the chunk parser.

\item {} 
\textbf{correct}: The correct set of chunks, as defined in the test corpus.

\end{itemize}

We will set up
an analogy between the correct set of chunks and a user's so-called ``information need'',
and between the set of returned chunks and a system's returned documents
(cf precision and recall, from \href{\#chap-data-intensive}{Chapter 4}).

During evaluation of a chunk parser, it is useful to flatten a chunk
structure into a tree consisting only of a root node and leaves:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }correct = nltk.chunk.tagstr2tree(
\pysrcmore{... }   \pysrcstring{"{[} the/DT little/JJ cat/NN {]} sat/VBD on/IN {[} the/DT mat/NN {]}"})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} correct.flatten()
\pysrcoutput{(S the/DT little/JJ cat/NN sat/VBD on/IN the/DT mat/NN)}}\end{alltt}

We run a chunker over this flattened data, and compare the
resulting chunked sentences with the originals, as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }grammar = r\pysrcstring{"NP: {\{}<PRP|DT|POS|JJ|CD|N.*>+{\}}"}
\pysrcprompt{>{}>{}> }cp = nltk.RegexpParser(grammar)
\pysrcprompt{>{}>{}> }tagged{\_}tokens = {[}(\pysrcstring{"the"}, \pysrcstring{"DT"}), (\pysrcstring{"little"}, \pysrcstring{"JJ"}), (\pysrcstring{"cat"}, \pysrcstring{"NN"}),
\pysrcmore{... }(\pysrcstring{"sat"}, \pysrcstring{"VBD"}), (\pysrcstring{"on"}, \pysrcstring{"IN"}), (\pysrcstring{"the"}, \pysrcstring{"DT"}), (\pysrcstring{"mat"}, \pysrcstring{"NN"}){]}
\pysrcprompt{>{}>{}> }chunkscore = nltk.chunk.ChunkScore()
\pysrcprompt{>{}>{}> }guess = cp.parse(correct.flatten())
\pysrcprompt{>{}>{}> }chunkscore.score(correct, guess)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} chunkscore
\pysrcoutput{ChunkParse score:}
\pysrcoutput{    Precision: 100.0{\%}}
\pysrcoutput{    Recall:    100.0{\%}}
\pysrcoutput{    F-Measure: 100.0{\%}}}\end{alltt}

\texttt{ChunkScore} is a class for scoring chunk parsers.  It can be used
to evaluate the output of a chunk parser, using precision, recall,
f-measure, missed chunks, and incorrect chunks.  It can also be used
to combine the scores from the parsing of multiple texts.  This is
quite useful if we are parsing a text one sentence at a time.  The
following program listing shows a typical use of the \texttt{ChunkScore}
class.  In this example, \texttt{chunkparser} is being tested on each
sentence from the Wall Street Journal tagged files.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }grammar = r\pysrcstring{"NP: {\{}<DT|JJ|NN>+{\}}"}
\pysrcprompt{>{}>{}> }cp = nltk.RegexpParser(grammar)
\pysrcprompt{>{}>{}> }chunkscore = nltk.chunk.ChunkScore()
\pysrcprompt{>{}>{}> }\pysrckeyword{for} file \pysrckeyword{in} nltk.corpus.treebank{\_}chunk.files(){[}:5{]}:
\pysrcmore{... }    \pysrckeyword{for} chunk{\_}struct \pysrckeyword{in} nltk.corpus.treebank{\_}chunk.chunked{\_}sents(file):
\pysrcmore{... }        test{\_}sent = cp.parse(chunk{\_}struct.flatten())
\pysrcmore{... }        chunkscore.score(chunk{\_}struct, test{\_}sent)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} chunkscore
\pysrcoutput{ChunkParse score:}
\pysrcoutput{    Precision:  42.3{\%}}
\pysrcoutput{    Recall:     29.9{\%}}
\pysrcoutput{    F-Measure:  35.0{\%}}}\end{alltt}

The overall results of the evaluation can be viewed by printing the
\texttt{ChunkScore}.  Each evaluation metric is also returned by an
accessor method: \texttt{precision{\linebreak[0]}({\linebreak[0]})}, \texttt{recall}, \texttt{f{\_}measure},
\texttt{missed}, and \texttt{incorrect}.  The \texttt{missed} and \texttt{incorrect}
methods can be especially useful when trying to improve the
performance of a chunk parser.  Here are the missed chunks:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} random \pysrckeyword{import} shuffle
\pysrcprompt{>{}>{}> }missed = chunkscore.missed()
\pysrcprompt{>{}>{}> }shuffle(missed)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} missed{[}:10{]}
\pysrcoutput{{[}(('A', 'DT'), ('Lorillard', 'NNP'), ('spokeswoman', 'NN')),}
\pysrcoutput{ (('even', 'RB'), ('brief', 'JJ'), ('exposures', 'NNS')),}
\pysrcoutput{ (('its', 'PRP{\$}'), ('Micronite', 'NN'), ('cigarette', 'NN'), ('filters', 'NNS')),}
\pysrcoutput{ (('30', 'CD'), ('years', 'NNS')),}
\pysrcoutput{ (('workers', 'NNS'),),}
\pysrcoutput{ (('preliminary', 'JJ'), ('findings', 'NNS')),}
\pysrcoutput{ (('Medicine', 'NNP'),),}
\pysrcoutput{ (('Consolidated', 'NNP'), ('Gold', 'NNP'), ('Fields', 'NNP'), ('PLC', 'NNP')),}
\pysrcoutput{ (('its', 'PRP{\$}'), ('Micronite', 'NN'), ('cigarette', 'NN'), ('filters', 'NNS')),}
\pysrcoutput{ (('researchers', 'NNS'),){]}}}\end{alltt}

Here are the incorrect chunks:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }incorrect = chunkscore.incorrect()
\pysrcprompt{>{}>{}> }shuffle(incorrect)
\pysrcoutput{>{}> print incorrect{[}:10{]}}
\pysrcoutput{{[}(('New', 'JJ'), ('York-based', 'JJ')),}
\pysrcoutput{ (('Micronite', 'NN'), ('cigarette', 'NN')),}
\pysrcoutput{ (('a', 'DT'), ('forum', 'NN'), ('likely', 'JJ')),}
\pysrcoutput{ (('later', 'JJ'),),}
\pysrcoutput{ (('preliminary', 'JJ'),),}
\pysrcoutput{ (('New', 'JJ'), ('York-based', 'JJ')),}
\pysrcoutput{ (('resilient', 'JJ'),),}
\pysrcoutput{ (('group', 'NN'),),}
\pysrcoutput{ (('the', 'DT'),),}
\pysrcoutput{ (('Micronite', 'NN'), ('cigarette', 'NN')){]}}}\end{alltt}
% Note: By default, only the first 100 missed chunks and the first
% 100 incorrect chunks will be remembered by the ``ChunkScore``.  You
% can tell ``ChunkScore`` to record more chunk examples with the
% ``max_fp_examples`` (maximum false positive examples) and the
% ``max_fn_examples`` (maximum false negative examples) keyword
% arguments to the ``ChunkScore`` constructor:
% 
%  >>> chunkscore = nltk.chunk.ChunkScore(max_fp_examples=1000,
%  ...                                    max_fn_examples=1000) 


%___________________________________________________________________________

\subsection{Exercises}
\label{id54}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
◑ \textbf{Chunker Evaluation:}
Carry out the following evaluation tasks for
any of the chunkers you have developed earlier.
(Note that most chunking corpora contain some internal
inconsistencies, such that any reasonable rule-based approach
will produce errors.)
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Evaluate your chunker on 100 sentences from a chunked corpus,
and report the precision, recall and F-measure.

\item {} 
Use the \texttt{chunkscore{\linebreak[0]}.{\linebreak[0]}missed{\linebreak[0]}({\linebreak[0]})} and \texttt{chunkscore{\linebreak[0]}.{\linebreak[0]}incorrect{\linebreak[0]}({\linebreak[0]})}
methods to identify the errors made by your chunker.  Discuss.

\item {} 
Compare the performance of your chunker to the baseline chunker
discussed in the evaluation section of this chapter.

\end{list}

\item {} 
★ \textbf{Transformation-Based Chunking:}
Apply the n-gram and Brill tagging methods to IOB chunk tagging.
Instead of assigning POS tags to words, here we will assign IOB tags
to the POS tags.  E.g. if the tag \texttt{DT} (determiner) often occurs
at the start of a chunk, it will be tagged \texttt{B} (begin).  Evaluate
the performance of these chunking methods relative to the regular
expression chunking methods covered in this chapter.

\end{list}
% TODO: amplify following example 


%___________________________________________________________________________

\subsection{Exercises}
\label{id55}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼
Pick one of the three chunk types in the CoNLL corpus.
Inspect the CoNLL corpus and try to observe any patterns in the POS tag sequences
that make up this kind of chunk.  Develop a simple chunker using
the regular expression chunker \texttt{nltk{\linebreak[0]}.{\linebreak[0]}RegexpParser}.
Discuss any tag sequences that are difficult to chunk reliably.

\item {} 
☼
An early definition of \emph{chunk} was the material that occurs between chinks.
Develop a chunker that starts by putting the whole sentence in a single
chunk, and then does the rest of its work solely by chinking.
Determine which tags (or tag sequences) are most likely to make up chinks
with the help of your own utility program.  Compare the performance and
simplicity of this approach relative to a chunker based entirely on
chunk rules.

\item {} 
◑
Develop a chunker for one of the chunk types in the CoNLL corpus using a
regular-expression based chunk grammar \texttt{RegexpChunk}.  Use any
combination of rules for chunking, chinking, merging or splitting.

\item {} 
◑ Sometimes a word is incorrectly tagged, e.g. the head noun in
``12/CD or/CC so/RB cases/VBZ''.  Instead of requiring manual correction of
tagger output, good chunkers are able to work with the erroneous
output of taggers.  Look for other examples of correctly chunked
noun phrases with incorrect tags.

\item {} 
★
We saw in the tagging chapter that it is possible to establish
an upper limit to tagging performance by looking for ambiguous n-grams,
n-grams that are tagged in more than one possible way in the training data.
Apply the same method to determine an upper bound on the performance
of an n-gram chunker.

\item {} 
★
Pick one of the three chunk types in the CoNLL corpus.  Write functions
to do the following tasks for your chosen type:
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
List all the tag sequences that occur with each instance of this chunk type.

\item {} 
Count the frequency of each tag sequence, and produce a ranked list in
order of decreasing frequency; each line should consist of an integer (the frequency)
and the tag sequence.

\item {} 
Inspect the high-frequency tag sequences.  Use these as the basis for
developing a better chunker.

\end{list}

\item {} 
★
The baseline chunker presented in the evaluation section tends to
create larger chunks than it should.  For example, the
phrase:
\texttt{{[}{\linebreak[0]}every{\linebreak[0]}/{\linebreak[0]}DT{\linebreak[0]} {\linebreak[0]}time{\linebreak[0]}/{\linebreak[0]}NN{\linebreak[0]}{]}{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}she{\linebreak[0]}/{\linebreak[0]}PRP{\linebreak[0]}{]}{\linebreak[0]} {\linebreak[0]}sees{\linebreak[0]}/{\linebreak[0]}VBZ{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}a{\linebreak[0]}/{\linebreak[0]}DT{\linebreak[0]} {\linebreak[0]}newspaper{\linebreak[0]}/{\linebreak[0]}NN{\linebreak[0]}{]}}
contains two consecutive chunks, and our baseline chunker will
incorrectly combine the first two: \texttt{{[}{\linebreak[0]}every{\linebreak[0]}/{\linebreak[0]}DT{\linebreak[0]} {\linebreak[0]}time{\linebreak[0]}/{\linebreak[0]}NN{\linebreak[0]} {\linebreak[0]}she{\linebreak[0]}/{\linebreak[0]}PRP{\linebreak[0]}{]}}.
Write a program that finds which of these chunk-internal tags
typically occur at the start of a chunk, then
devise one or more rules that will split up these chunks.
Combine these with the existing baseline chunker and
re-evaluate it, to see if you have discovered an improved baseline.

\item {} 
★
Develop an \docutilsrolecategory{np} chunker that converts POS-tagged text into a list of
tuples, where each tuple consists of a verb followed by a sequence of
noun phrases and prepositions,
e.g. \texttt{the{\linebreak[0]} {\linebreak[0]}little{\linebreak[0]} {\linebreak[0]}cat{\linebreak[0]} {\linebreak[0]}sat{\linebreak[0]} {\linebreak[0]}on{\linebreak[0]} {\linebreak[0]}the{\linebreak[0]} {\linebreak[0]}mat} becomes \texttt{(\pysrcstring{'{\linebreak[0]}sat{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}on{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}NP{\linebreak[0]}'})}...

\item {} 
★
The Penn Treebank contains a section of tagged Wall Street Journal text
that has been chunked into noun phrases.  The format uses square brackets,
and we have encountered it several times during this chapter.
The Treebank corpus can be accessed using:
\texttt{\pysrckeyword{for} {\linebreak[0]}sent{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}treebank{\_}chunk{\linebreak[0]}.{\linebreak[0]}chunked{\_}sents{\linebreak[0]}({\linebreak[0]}file{\linebreak[0]})}.  These are flat trees,
just as we got using \texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}conll2000{\linebreak[0]}.{\linebreak[0]}chunked{\_}sents{\linebreak[0]}({\linebreak[0]})}.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
The functions \texttt{nltk{\linebreak[0]}.{\linebreak[0]}tree{\linebreak[0]}.{\linebreak[0]}pprint{\linebreak[0]}({\linebreak[0]})} and \texttt{nltk{\linebreak[0]}.{\linebreak[0]}chunk{\linebreak[0]}.{\linebreak[0]}tree2conllstr{\linebreak[0]}({\linebreak[0]})}
can be used to create Treebank and IOB strings from a tree.
Write functions \texttt{chunk2brackets{\linebreak[0]}({\linebreak[0]})} and \texttt{chunk2iob{\linebreak[0]}({\linebreak[0]})} that take a single
chunk tree as their sole argument, and return the required multi-line string
representation.

\item {} 
Write command-line conversion utilities \texttt{bracket2iob{\linebreak[0]}.{\linebreak[0]}py} and \texttt{iob2bracket{\linebreak[0]}.{\linebreak[0]}py}
that take a file in Treebank or CoNLL format (resp) and convert it to the other
format.  (Obtain some raw Treebank or CoNLL data from the NLTK Corpora, save it
to a file, and then use \texttt{\pysrckeyword{for} {\linebreak[0]}line{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}open{\linebreak[0]}({\linebreak[0]}filename{\linebreak[0]})} to access it from Python.)

\end{list}

\end{list}
\hypertarget{n-gram-chunking}{}

%___________________________________________________________________________

\section{N-Gram Chunking}
\label{id56}
\label{n-gram-chunking}

Our approach to chunking has been to try to detect structure based on
the part-of-speech tags.  We have seen that the IOB format represents
this extra structure using another kind of tag.  The question arises
as to whether we could use the same \emph{n}-gram tagging methods we saw in
\href{\#chap-tag}{Chapter 3}, applied to a different vocabulary. In this case,
rather than trying to determine the correct part-of-speech tag, given
a word, we are trying to determine the correct chunk tag, given a
part-of-speech tag.

The first step is to get the \texttt{word{\linebreak[0]},{\linebreak[0]}tag{\linebreak[0]},{\linebreak[0]}chunk} triples from the
CoNLL 2000 corpus and map these to \texttt{tag{\linebreak[0]},{\linebreak[0]}chunk} pairs:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }chunk{\_}data = {[}{[}(t,c) \pysrckeyword{for} w,t,c \pysrckeyword{in} nltk.chunk.tree2conlltags(chtree){]}
\pysrcmore{... }             \pysrckeyword{for} chtree \pysrckeyword{in} nltk.corpus.conll2000.chunked{\_}sents(\pysrcstring{'train.txt'}){]}}\end{alltt}

We will now train two \emph{n}-gram taggers over this data.


%___________________________________________________________________________

\subsection{A Unigram Chunker}
\label{a-unigram-chunker}

To start off, we train and score a \index{unigram chunker}\textbf{unigram chunker} on the above data, just as if
it was a tagger:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }unigram{\_}chunker = nltk.UnigramTagger(chunk{\_}data)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.tag.accuracy(unigram{\_}chunker, chunk{\_}data)
\pysrcoutput{0.781378851068}}\end{alltt}

This chunker does reasonably well.  Let's look at the errors it makes.
Consider the opening phrase of the first sentence of the CONLL chunking
data, here shown with part-of-speech tags:
\begin{quote}

Confidence/NN in/IN the/DT pound/NN is/VBZ widely/RB expected/VBN
to/TO take/VB another/DT sharp/JJ dive/NN
\end{quote}

We can try out the unigram chunker on this first sentence by creating
some ``tokens'' using \texttt{{[}{\linebreak[0]}t{\linebreak[0]} \pysrckeyword{for} {\linebreak[0]}t{\linebreak[0]},{\linebreak[0]}c{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}chunk{\_}data{\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}{]}{\linebreak[0]}{]}}, then running
our chunker over them using \texttt{list{\linebreak[0]}({\linebreak[0]}unigram{\_}chunker{\linebreak[0]}.{\linebreak[0]}tag{\linebreak[0]}({\linebreak[0]}tokens{\linebreak[0]}){\linebreak[0]})}.
The unigram chunker only looks at the tags, and tries to add chunk
tags.  Here is what it comes up with:
\begin{quote}

NN/I-NP IN/B-PP DT/B-NP NN/I-NP VBZ/B-VP RB/O VBN/I-VP TO/B-PP
VB/I-VP DT/B-NP JJ/I-NP NN/I-NP
\end{quote}

Notice that it tags all instances of \texttt{NN} with \texttt{I{\linebreak[0]}-{\linebreak[0]}NP}, because
nouns usually do not appear at the beginning of noun phrases in
the training data.  Thus, the first noun \texttt{Confidence{\linebreak[0]}/{\linebreak[0]}NN} is
tagged incorrectly.  However, \texttt{pound{\linebreak[0]}/{\linebreak[0]}NN} and \texttt{dive{\linebreak[0]}/{\linebreak[0]}NN} are
correctly tagged as \texttt{I{\linebreak[0]}-{\linebreak[0]}NP}; they are not in the initial position
that should be tagged \texttt{B{\linebreak[0]}-{\linebreak[0]}NP}.  The chunker incorrectly tags
\texttt{widely{\linebreak[0]}/{\linebreak[0]}RB} as \texttt{O}, and it incorrectly tags the
infinitival \texttt{to{\linebreak[0]}/{\linebreak[0]}TO} as \texttt{B{\linebreak[0]}-{\linebreak[0]}PP}, as if it was a preposition starting a
prepositional phrase.


%___________________________________________________________________________

\subsection{A Bigram Chunker (incomplete)}
\label{a-bigram-chunker-incomplete}

{[}Why these problems might go away if we look at the previous chunk tag?{]}

Let's run a bigram chunker:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }bigram{\_}chunker = nltk.BigramTagger(chunk{\_}data, backoff=unigram{\_}chunker)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.tag.accuracy(bigram{\_}chunker, chunk{\_}data)
\pysrcoutput{0.893220987404}}\end{alltt}

We can run the bigram chunker over the same sentence as before
using \texttt{list{\linebreak[0]}({\linebreak[0]}bigram{\_}chunker{\linebreak[0]}.{\linebreak[0]}tag{\linebreak[0]}({\linebreak[0]}tokens{\linebreak[0]}){\linebreak[0]})}.
Here is what it comes up with:
\begin{quote}

NN/B-NP IN/B-PP DT/B-NP NN/I-NP VBZ/B-VP RB/I-VP VBN/I-VP TO/I-VP
VB/I-VP DT/B-NP JJ/I-NP NN/I-NP
\end{quote}

This is 100{\%} correct.


%___________________________________________________________________________

\subsection{Exercises}
\label{id57}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
◑
The bigram chunker scores about 90{\%} accuracy.
Study its errors and try to work out why it doesn't get 100{\%} accuracy.

\item {} 
◑
Experiment with trigram chunking.  Are you
able to improve the performance any more?

\item {} 
★
An n-gram chunker can use information other than the current
part-of-speech tag and the \docutilsrolemath{n-1} previous chunk tags.
Investigate other models of the context, such as
the \docutilsrolemath{n-1} previous part-of-speech tags, or some combination of
previous chunk tags along with previous and following part-of-speech tags.

\item {} 
★
Consider the way an n-gram tagger uses recent tags to inform its tagging choice.
Now observe how a chunker may re-use this sequence information.  For example,
both tasks will make use of the information that nouns tend to follow adjectives
(in English).  It would appear that the same information is being maintained in
two places.  Is this likely to become a problem as the size of the rule sets grows?
If so, speculate about any ways that this problem might be addressed.

\end{list}


%___________________________________________________________________________

\section{Cascaded Chunkers}
\label{cascaded-chunkers}

So far, our chunk structures have been relatively flat.  Trees consist
of tagged tokens, optionally grouped under a chunk node such as
\texttt{NP}.  However, it is possible to build chunk structures of
arbitrary depth, simply by creating a multi-stage chunk grammar.
These stages are processed in the order
that they appear.  The patterns in later stages can refer to a mixture
of part-of-speech tags and chunk types.  \href{\#cascaded-chunker}{Listing 6.7} has
patterns for noun phrases, prepositional phrases, verb phrases, and
sentences.
This is a four-stage chunk grammar, and can be used to create
structures having a depth of at most four.
\hypertarget{cascaded-chunker}{}
\begin{pylisting}
\begin{alltt}\small\textbf{grammar = r\pysrcstring{"{}"{}"}
\pysrcstring{  NP: {\{}<DT|JJ|NN.*>+{\}}       {\#} Chunk sequences of DT, JJ, NN}
\pysrcstring{  PP: {\{}<IN><NP>{\}}            {\#} Chunk prepositions followed by NP}
\pysrcstring{  VP: {\{}<VB.*><NP|PP|S>+{\$}{\}}   {\#} Chunk rightmost verbs and arguments/adjuncts}
\pysrcstring{  S:  {\{}<NP><VP>{\}}            {\#} Chunk NP, VP}
\pysrcstring{  "{}"{}"}
cp = nltk.RegexpParser(grammar)
tagged{\_}tokens = {[}(\pysrcstring{"Mary"}, \pysrcstring{"NN"}), (\pysrcstring{"saw"}, \pysrcstring{"VBD"}), (\pysrcstring{"the"}, \pysrcstring{"DT"}), (\pysrcstring{"cat"}, \pysrcstring{"NN"}),
    (\pysrcstring{"sit"}, \pysrcstring{"VB"}), (\pysrcstring{"on"}, \pysrcstring{"IN"}), (\pysrcstring{"the"}, \pysrcstring{"DT"}), (\pysrcstring{"mat"}, \pysrcstring{"NN"}){]}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} cp.parse(tagged{\_}tokens)
\pysrcoutput{(S}
\pysrcoutput{  (NP Mary/NN)}
\pysrcoutput{  saw/VBD}
\pysrcoutput{  (S}
\pysrcoutput{    (NP the/DT cat/NN)}
\pysrcoutput{    (VP sit/VB (PP on/IN (NP the/DT mat/NN)))))}}\end{alltt}
\caption{A Chunker that Handles NP, PP, VP and S}
\vspace{1ex}
\hrule
\end{pylisting}

Unfortunately this result misses the \docutilsrolecategory{vp} headed by \docutilsroleexample{saw}.  It has
other shortcomings too.  Let's see what happens when we apply this
chunker to a sentence having deeper nesting.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tagged{\_}tokens = {[}(\pysrcstring{"John"}, \pysrcstring{"NNP"}), (\pysrcstring{"thinks"}, \pysrcstring{"VBZ"}), (\pysrcstring{"Mary"}, \pysrcstring{"NN"}),
\pysrcmore{... }    (\pysrcstring{"saw"}, \pysrcstring{"VBD"}), (\pysrcstring{"the"}, \pysrcstring{"DT"}), (\pysrcstring{"cat"}, \pysrcstring{"NN"}), (\pysrcstring{"sit"}, \pysrcstring{"VB"}),
\pysrcmore{... }    (\pysrcstring{"on"}, \pysrcstring{"IN"}), (\pysrcstring{"the"}, \pysrcstring{"DT"}), (\pysrcstring{"mat"}, \pysrcstring{"NN"}){]}
\pysrcprompt{>{}>{}> }\pysrckeyword{print} cp.parse(tagged{\_}tokens)
\pysrcoutput{(S}
\pysrcoutput{  (NP John/NNP)}
\pysrcoutput{  thinks/VBZ}
\pysrcoutput{  (NP Mary/NN)}
\pysrcoutput{  saw/VBD}
\pysrcoutput{  (S}
\pysrcoutput{    (NP the/DT cat/NN)}
\pysrcoutput{    (VP sit/VB (PP on/IN (NP the/DT mat/NN)))))}}\end{alltt}

The solution to these problems is to get the chunker to loop over its
patterns: after trying all of them, it repeats the process.
We add an optional second argument \texttt{loop} to specify the number
of times the set of patterns should be run:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }cp = nltk.RegexpParser(grammar, loop=2)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} cp.parse(tagged{\_}tokens)
\pysrcoutput{(S}
\pysrcoutput{  (NP John/NNP)}
\pysrcoutput{  thinks/VBZ}
\pysrcoutput{  (S}
\pysrcoutput{    (NP Mary/NN)}
\pysrcoutput{    (VP}
\pysrcoutput{      saw/VBD}
\pysrcoutput{      (S}
\pysrcoutput{        (NP the/DT cat/NN)}
\pysrcoutput{        (VP sit/VB (PP on/IN (NP the/DT mat/NN)))))))}}\end{alltt}

This cascading process enables us to create deep structures.  However,
creating and debugging a cascade is quite difficult, and there comes
a point where it is more effective to do full parsing (see \href{\#chap-parse}{Chapter 7}).


%___________________________________________________________________________

\section{Shallow Interpretation}
\label{shallow-interpretation}

The main form of shallow semantic interpretation that we will consider
is \index{Information Extraction}\textbf{Information Extraction}. This refers to the task of converting
\index{unstructured data}\textbf{unstructured data} (e.g., unrestricted text) or \index{semi-structured
data}\textbf{semi-structured
data} (e.g., web pages marked up with HTML) into \index{structured
data}\textbf{structured
data} (e.g., tables in a relational database). For example, let's
suppose we are given a text containing the fragment \href{\#ie1}{(12)}, and let's
also suppose we are trying to find pairs of entities \emph{X} and \emph{Y} that
stand in the relation 'organization \emph{X} is located in location \emph{Y}'.
\hypertarget{ie1}{}\begin{itemize}
\item[(12)] 
... said William Gale, an economist at the Brookings Institution,
the research group in Washington.
\end{itemize}

{\noindent} As a result of processing this text, we should be able to add
the pair 〈\emph{Brookings Institution}, \emph{Washington}〉 to
this relation. As we will see shortly, Information Extraction proceeds
on the assumption that we are only looking for specific sorts of
information, and these have been decided in advance. This limitation
has been a necessary concession to allow the robust processing of
unrestricted text.

Potential applications of Information Extraction are
many, and include business intelligence, resume harvesting, media
analysis, sentiment detection, patent search, and email scanning. A
particularly important area of current research involves the attempt
to extract structured data out of electronically-available scientific
literature, most notably in the domain of biology and medicine.

Information Extraction is usually broken down into at least two major
steps: \index{Named Entity Recognition}\textbf{Named Entity Recognition} and \index{Relation
Extraction}\textbf{Relation
Extraction}. Named Entities (NEs) are usually taken to be noun phrases
that denote specific types of individuals such as organizations,
persons, dates, and so on. Thus, we
might use the following XML annotations to mark-up the NEs in \href{\#ie1}{(12)}:
\hypertarget{ie2}{}\begin{itemize}
\item[(13)] 
... said {\textless}ne type='PERSON'{\textgreater}William Gale{\textless}/ne{\textgreater}, an economist at
the {\textless}ne type='ORGANIZATION'{\textgreater}Brookings Institution{\textless}/ne{\textgreater},
the research group in {\textless}ne type='LOCATION'{\textgreater}Washington{\textless}ne{\textgreater}.
\end{itemize}

How do we go about identifying NEs? Our first thought might be
that we could look up candidate expressions in an appropriate list of
names. For example, in the case of locations, we might  try using a
resource such as the \href{http://www.alexandria.ucsb.edu/gazetteer/}{Alexandria Gazetteer}. Depending on the nature
of our input data, this may be adequate — such a gazetteer is likely
to have good coverage of international cities and many locations in
the U.S.A., but will probably be missing the names of obscure villages
in remote regions. However, a list of names for people or organizations
will probably have poor coverage. New organizations, and new names for
them, are coming into existence every day, so if we are trying to deal
with contemporary newswire or blog entries, say, it is unlikely that
we will be able to recognize many of the NEs by using gazetteer lookup.

A second consideration is that many NE terms are ambiguous. Thus
\docutilsroleexample{May} and \docutilsroleexample{North} are likely to be parts of NEs for DATE
and LOCATION, respectively, but could both be part of a PERSON NE;
conversely \docutilsroleexample{Christian Dior} looks like a PERSON NE but is more
likely to be of type ORGANIZATION. A term like \docutilsroleexample{Yankee} will be
ordinary modifier in some contexts, but will be marked as an NE of
type ORGANIZATION in the phrase \docutilsroleexample{Yankee infielders}. To summarize,
we cannot reliably detect NEs by looking them up in a gazetteer,
and it is also hard to develop rules that will correctly recognize
ambiguous NEs on the basis of their context of
occurrence. Although lookup may contribute to a solution, most
contemporary approaches to Named Entity Recognition treat it as a
statistical classification task that requires training data for good
performance. This task is facilitated by adopting an appropriate data
representation, such as the IOB tags that we saw being deployed in
the CoNLL chunk data (\href{\#chap-chunk}{Chapter 6}). For example, here are a
representative few lines from the CONLL 2002 (\texttt{conll2002}) Dutch
training data:
\begin{quote}{\ttfamily \raggedright \noindent
\small
Eddy~N~B-PER~\\
Bonte~N~I-PER~\\
is~V~O~\\
woordvoerder~N~O~\\
van~Prep~O~\\
diezelfde~Pron~O~\\
Hogeschool~N~B-ORG~\\
.~Punc~O
}\end{quote}

{\noindent} As noted before, in this representation, there is one token
per line, each with its part-of-speech tag and its NE tag.  When
NEs have been identified in a text, we then want to extract
relations that hold between them. As indicated earlier, we will
typically be looking for relations between specified types of
NE. One way of approaching this task is to initially look for all
triples of the form \emph{X}, α, \emph{Y}, where \emph{X} and \emph{Y} are NEs
of the required types, and α is the string of words that
intervenes between \emph{X} and \emph{Y}. We can then use regular expressions to
pull out just those instances of α that express the relation
that we are looking for. The following example searches for strings
that contain the word \docutilsroleexample{in}. The special character expression
\texttt{({\linebreak[0]}?{\linebreak[0]}!{\linebreak[0]}{\textbackslash}{\linebreak[0]}b{\linebreak[0]}.{\linebreak[0]}+{\linebreak[0]}ing{\linebreak[0]}{\textbackslash}{\linebreak[0]}b{\linebreak[0]})} is a negative lookahead condition that allows us to
disregard strings such as \docutilsroleexample{success in supervising the transition
of}, where \docutilsroleexample{in} is followed by a gerundive verb.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }IN = re.compile(r\pysrcstring{'.*{\textbackslash}bin{\textbackslash}b(?!{\textbackslash}b.+ing{\textbackslash}b)'})
\pysrcprompt{>{}>{}> }\pysrckeyword{for} doc \pysrckeyword{in} nltk.corpus.ieer.parsed{\_}docs(\pysrcstring{'NYT{\_}19980315'}):
\pysrcmore{... }    \pysrckeyword{for} rel \pysrckeyword{in} nltk.sem.extract{\_}rels(\pysrcstring{'ORG'}, \pysrcstring{'LOC'}, doc, pattern = IN):
\pysrcmore{... }        \pysrckeyword{print} nltk.sem.show{\_}raw{\_}rtuple(rel)
\pysrcoutput{{[}ORG: 'WHYY'{]} 'in' {[}LOC: 'Philadelphia'{]}}
\pysrcoutput{{[}ORG: 'McGlashan {\&}AMP; Sarrail'{]} 'firm in' {[}LOC: 'San Mateo'{]}}
\pysrcoutput{{[}ORG: 'Freedom Forum'{]} 'in' {[}LOC: 'Arlington'{]}}
\pysrcoutput{{[}ORG: 'Brookings Institution'{]} ', the research group in' {[}LOC: 'Washington'{]}}
\pysrcoutput{{[}ORG: 'Idealab'{]} ', a self-described business incubator based in' {[}LOC: 'Los Angeles'{]}}
\pysrcoutput{{[}ORG: 'Open Text'{]} ', based in' {[}LOC: 'Waterloo'{]}}
\pysrcoutput{{[}ORG: 'WGBH'{]} 'in' {[}LOC: 'Boston'{]}}
\pysrcoutput{{[}ORG: 'Bastille Opera'{]} 'in' {[}LOC: 'Paris'{]}}
\pysrcoutput{{[}ORG: 'Omnicom'{]} 'in' {[}LOC: 'New York'{]}}
\pysrcoutput{{[}ORG: 'DDB Needham'{]} 'in' {[}LOC: 'New York'{]}}
\pysrcoutput{{[}ORG: 'Kaplan Thaler Group'{]} 'in' {[}LOC: 'New York'{]}}
\pysrcoutput{{[}ORG: 'BBDO South'{]} 'in' {[}LOC: 'Atlanta'{]}}
\pysrcoutput{{[}ORG: 'Georgia-Pacific'{]} 'in' {[}LOC: 'Atlanta'{]}}}\end{alltt}

{\noindent} Searching for the keyword works \docutilsroleexample{in} reasonably well,
though it will also retrieve false positives such as \texttt{{[}{\linebreak[0]}ORG{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}House
Transportation{\linebreak[0]} {\linebreak[0]}Committee{\linebreak[0]}{]}{\linebreak[0]} {\linebreak[0]},{\linebreak[0]} {\linebreak[0]}secured{\linebreak[0]} {\linebreak[0]}the{\linebreak[0]} {\linebreak[0]}most{\linebreak[0]} {\linebreak[0]}money{\linebreak[0]} \pysrckeyword{in} {\linebreak[0]}the{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}LOC{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}New
York{\linebreak[0]}{]}}; there is unlikely to be simple string-based method of
excluding filler strings such as this.
% TODO fix processing of tagged corpora 
% As shown above, the ``conll2002`` corpus contains not just |NE|
% annotation but also part-of-speech tags. In principle, this allows us
% to devise patterns that are sensitive to these tags. 
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }vnv = \pysrcstring{"{}"{}"}
\pysrcmore{... }\pysrcstring{(}
\pysrcmore{... }\pysrcstring{is/V|}
\pysrcmore{... }\pysrcstring{was/V|}
\pysrcmore{... }\pysrcstring{werd/V|}
\pysrcmore{... }\pysrcstring{wordt/V}
\pysrcmore{... }\pysrcstring{)}
\pysrcmore{... }\pysrcstring{.*}
\pysrcmore{... }\pysrcstring{van/Prep}
\pysrcmore{... }\pysrcstring{"{}"{}"}
\pysrcprompt{>{}>{}> }VAN = re.compile(vnv, re.VERBOSE)
\pysrcprompt{>{}>{}> }\pysrckeyword{for} r \pysrckeyword{in} nltk.sem.extract{\_}rels(\pysrcstring{'PER'}, \pysrcstring{'ORG'}, corpus=\pysrcstring{'conll2002-ned'}, pattern=VAN):
\pysrcmore{... }    \pysrckeyword{print} show{\_}tuple(r)}\end{alltt}


%___________________________________________________________________________

\section{Conclusion}
\label{id58}

In this chapter we have explored efficient and robust methods that can
identify linguistic structures in text.  Using only part-of-speech
information for words in the local context, a ``chunker'' can
successfully identify simple structures such as noun phrases and verb
groups.  We have seen how chunking methods extend the same lightweight
methods that were successful in tagging.  The resulting structured
information is useful in information extraction tasks and in the
description of the syntactic environments of words.  The latter will
be invaluable as we move to full parsing.

There are a surprising number of ways to chunk a sentence using
regular expressions.  The patterns can add, shift and remove chunks in
many ways, and the patterns can be sequentially ordered in many ways.
One can use a small number of very complex rules, or a long sequence
of much simpler rules.  One can hand-craft a collection of rules, and
one can write programs to analyze a chunked corpus to help in the
development of such rules.  The process is painstaking, but generates
very compact chunkers that perform well and that transparently encode
linguistic knowledge.

It is also possible to chunk a sentence using the techniques of n-gram
tagging.  Instead of assigning part-of-speech tags to words, we assign
IOB tags to the part-of-speech tags.  Bigram tagging turned out to be
particularly effective, as it could be sensitive to the chunk tag on
the previous word.  This statistical approach requires far less effort
than rule-based chunking, but creates large models and delivers few
linguistic insights.

Like tagging, chunking cannot be done perfectly.  For example, as
pointed out by \cite{abney1996pst}, we cannot correctly analyze the structure
of the sentence \emph{I turned off the spectroroute} without knowing the
meaning of \emph{spectroroute}; is it a kind of road or a type of device?
Without knowing this, we cannot tell whether \emph{off} is part of a
prepositional phrase indicating direction (tagged \texttt{B{\linebreak[0]}-{\linebreak[0]}PP}), or
whether \emph{off} is part of the verb-particle construction \emph{turn off}
(tagged \texttt{I{\linebreak[0]}-{\linebreak[0]}VP}).

A recurring theme of this chapter has been \index{diagnosis}\textbf{diagnosis}.  The simplest
kind is manual, when we inspect the tracing output of a chunker and
observe some undesirable behavior that we would like to fix.
Sometimes we discover cases where we cannot hope to get the correct
answer because the part-of-speech tags are too impoverished and do not
give us sufficient information about the lexical item.  A second
approach is to write utility programs to analyze the training data,
such as counting the number of times a given part-of-speech tag occurs
inside and outside an \docutilsrolecategory{np} chunk.  A third approach is to evaluate the
system against some gold standard data to obtain an overall
performance score.  We can even use this to parameterize the system,
specifying which chunk rules are used on a given run, and tabulating
performance for different parameter combinations.  Careful use of
these diagnostic methods permits us to optimize the performance of our
system.  We will see this theme emerge again later in chapters dealing
with other topics in natural language processing.


%___________________________________________________________________________

\section{Further Reading}
\label{id60}

For more examples of chunking with NLTK, please see the
guide at \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}chunk{\linebreak[0]}.{\linebreak[0]}html}.

The popularity of chunking is due in great part to pioneering work by
Abney e.g., \cite{abney1996pst}. Abney's Cass chunker is available at
\href{http://www.vinartus.net/spa/97a.pdf}{http://www.vinartus.net/spa/97a.pdf}

The word \index{chink}\textbf{chink} initially meant a sequence of stopwords,
according to a 1975 paper by Ross and Tukey \cite{abney1996pst}.

The IOB format (or sometimes  \index{BIO Format}\textbf{BIO Format}) was developed for
\docutilsrolecategory{np} chunking by \cite{ramshaw1995tcu}, and was used for the shared \docutilsrolecategory{np}
bracketing task run by the \emph{Conference on Natural Language Learning}
(CoNLL) in 1999.  The same format was
adopted by CoNLL 2000 for annotating a section of Wall Street
Journal text as part of a shared task on \docutilsrolecategory{np} chunking.

Section 13.5 of \cite{jurafskymartin2008} contains a discussion of chunking.
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
\hypertarget{chap-parse}{}\setcounter{chapter}{6}

%___________________________________________________________________________

\chapter{Context Free Grammars and Parsing}
\label{context-free-grammars-and-parsing}
\label{chap-parse}
% TODO: explain that cfg grammars must not mix lexical and phrasal RHS 
% TODO: explain why RHS can't have a sequence of terminals 
% TODO: motivate or omit mathematical notation for formalizing CFGs 
% TODO: include overview of cfg.Grammar and cfg.Production in 8.4 
% TODO: put dependency grammar material in later chapter? 
% TODO: make URLs clickable in the HTML version 


%___________________________________________________________________________

\section{Introduction}
\label{id65}

Early experiences with the kind of grammar taught in school are sometimes perplexing.
Your written work might have been graded by a teacher who red-lined all
the grammar errors they wouldn't put up with.
Like the plural pronoun or the dangling preposition in the last sentence,
or sentences like this one that lack a main verb.
If you learnt English as a second language, you might have found it difficult
to discover which of these errors need to be fixed (or \emph{needs} to be fixed?).
Correct punctuation is an obsession for many writers and editors.
It is easy to find cases where changing punctuation changes meaning.
In the following example, the interpretation of a relative clause as
restrictive or non-restrictive depends on the presence of commas alone:
\begin{itemize}
\item[(14)] \hypertarget{rest}{}\begin{itemize}
\item[a.] 
The presidential candidate, who was extremely popular,
smiled broadly.
\end{itemize}
\hypertarget{nonrest}{}\begin{itemize}
\item[b.] 
The presidential candidate who was extremely popular smiled broadly.
\end{itemize}
\end{itemize}

In \href{\#rest}{(14a)}, we assume there is just one presidential candidate, and say
two things about her: that she was popular and that she smiled. In
\href{\#nonrest}{(14b)}, on the other hand, we use the description \docutilsroleexample{who was extremely
popular} as a means of identifying which of several possible
candidates we are referring to.

It is clear that some of these rules are important.
However, others seem to be vestiges of antiquated style.
Consider the injunction that \docutilsroleexample{however}
— when used to mean \emph{nevertheless} —
must not appear at the start of a sentence.
Pullum argues that Strunk and White \cite{strunkwhite1999} were
merely insisting that English usage should conform to ``an utterly
unimportant minor statistical detail of style concerning adverb
placement in the literature they knew'' \cite{pullum2005however}.
This is a case where, a \docutilsroleemphasis{descriptive} observation about language use became
a \docutilsroleemphasis{prescriptive} requirement.  In NLP we usually discard such prescriptions,
and use grammar to formalize observations about language as it is used,
particularly as it is used in corpora.

In this chapter we present the fundamentals of syntax, focusing on constituency
and tree representations, before describing the formal notation of context free
grammar.  Next we present parsers as an automatic way to associate syntactic
structures with sentences.  Finally, we give a detailed presentation of simple
top-down and bottom-up parsing algorithms available in NLTK.
Before launching into the theory we present some more naive observations about
grammar, for the benefit of readers who do not have a background in linguistics.
\hypertarget{sec-more-observations-about-grammar}{}

%___________________________________________________________________________

\section{More Observations about Grammar}
\label{more-observations-about-grammar}
\label{sec-more-observations-about-grammar}

Another function of a grammar is to explain our observations
about ambiguous sentences.  Even when the individual words are
unambiguous, we can put them together to create ambiguous sentences,
as in \href{\#ambiguous}{(15b)}.
\hypertarget{ambiguous}{}\begin{itemize}
\item[(15)] \begin{itemize}
\item[a.] 
Fighting animals could be dangerous.
\end{itemize}
\begin{itemize}
\item[b.] 
Visiting relatives can be tiresome.
\end{itemize}
\end{itemize}

A grammar will be able to assign two structures to each sentence,
accounting for the two possible interpretations.

Perhaps another kind of syntactic variation, word order, is easier to
understand.  We know that the two sentences \docutilsroleexample{Kim likes Sandy} and
\docutilsroleexample{Sandy likes Kim} have different meanings, and that \docutilsroleexample{likes Sandy
Kim} is simply ungrammatical.  Similarly, we know that the
following two sentences are equivalent:
\begin{itemize}
\item[(16)] \begin{itemize}
\item[a.] 
The farmer \emph{loaded} the cart with sand
\end{itemize}
\begin{itemize}
\item[b.] 
The farmer \emph{loaded} sand into the cart
\end{itemize}
\end{itemize}

However, consider the semantically similar verbs \docutilsroleexample{filled} and \docutilsroleexample{dumped}.
Now the word order cannot be altered (ungrammatical sentences are
prefixed with an asterisk.)
\begin{itemize}
\item[(17)] \begin{itemize}
\item[a.] 
The farmer \emph{filled} the cart with sand
\end{itemize}
\begin{itemize}
\item[b.] 
*The farmer \emph{filled} sand into the cart
\end{itemize}
\begin{itemize}
\item[c.] 
*The farmer \emph{dumped} the cart with sand
\end{itemize}
\begin{itemize}
\item[d.] 
The farmer \emph{dumped} sand into the cart
\end{itemize}
\end{itemize}

A further notable fact is that we have no difficulty accessing the meaning of
sentences we have never encountered before.  It is not difficult to concoct an
entirely novel sentence, one that has probably never been used before
in the history of the language, and yet all speakers of the language
will agree about its meaning.  In fact, the set of possible sentences is
infinite, given that there is no upper bound on length.
Consider the following passage from a children's story, containing a
rather impressive sentence:
\begin{quote}

You can imagine Piglet's joy when at last the ship came in sight of
him. In after-years he liked to think that he had been in Very
Great Danger during the Terrible Flood, but the only danger he had
really been in was the last half-hour of his imprisonment, when
Owl, who had just flown up, sat on a branch of his tree to comfort
him, and told him a very long story about an aunt who had once laid
a seagull's egg by mistake, and the story went on and on, rather
like this sentence, until Piglet who was listening out of his
window without much hope, went to sleep quietly and naturally,
slipping slowly out of the window towards the water until he was
only hanging on by his toes, at which moment, luckily, a sudden
loud squawk from Owl, which was really part of the story, being
what his aunt said, woke the Piglet up and just gave him time to
jerk himself back into safety and say, ``How interesting, and did
she?''  when -{}- well, you can imagine his joy when at last he saw
the good ship, Brain of Pooh (Captain, C. Robin; 1st Mate, P. Bear)
coming over the sea to rescue him...  (from A.A. Milne \emph{In which
Piglet is Entirely Surrounded by Water})
\end{quote}

Our ability to produce and understand entirely new sentences, of
arbitrary length, demonstrates that the set of well-formed sentences in
English is infinite.  The same case can be made for any human language.

This chapter presents grammars and parsing, as the formal and
computational methods for investigating and modeling the linguistic
phenomena we have been touching on (or tripping over).
As we shall see, patterns of well-formedness and ill-formedness in a
sequence of words can be understood with respect to the underlying
\index{phrase structure}\textbf{phrase structure} of the sentences.  We can develop formal
models of these structures using grammars and parsers.
As before, the motivation is natural language \emph{understanding}.  How
much more of the meaning of a text can we access when we can reliably
recognize the linguistic structures it contains?  Having read in a
text, can a program 'understand' it enough to be able to answer simple
questions about ``what happened'' or ``who did what to whom.''  Also as
before, we will develop simple programs to process annotated corpora
and perform useful tasks.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Remember that our program samples assume you
begin your interactive session or your program with: \texttt{\pysrckeyword{import} {\linebreak[0]}nltk{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}re{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}pprint}
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\section{What's the Use of Syntax?}
\label{what-s-the-use-of-syntax}

Earlier chapters focused on words: how to identify them, how to
analyze their morphology, and how to assign them to classes via
part-of-speech tags. We have also seen how to identify recurring
sequences of words (i.e. n-grams). Nevertheless, there seem to be
linguistic regularities that cannot be described simply in terms of
n-grams.

In this section we will see why it is useful to have some
kind of syntactic representation of sentences.  In particular, we will
see that there are systematic aspects of meaning that are much easier
to capture once we have established a level of syntactic structure.
\hypertarget{sec-syntactic-ambiguity}{}

%___________________________________________________________________________

\subsection{Syntactic Ambiguity}
\label{syntactic-ambiguity}
\label{sec-syntactic-ambiguity}

We have seen that sentences can be ambiguous.  If we overheard someone
say \docutilsroleexample{I went to the bank}, we wouldn't know whether it was
a river bank or a financial institution.  This ambiguity concerns
the meaning of the word \docutilsroleexample{bank}, and is a kind of \index{lexical
ambiguity}\textbf{lexical
ambiguity}.

However, other kinds of ambiguity cannot be explained in terms of
ambiguity of specific words.  Consider a phrase involving
an adjective with a conjunction:
\docutilsroleexample{old men and women}.
Does \docutilsroleexample{old} have wider scope than \docutilsroleexample{and}, or is it the other way
round? In fact, both interpretations are possible, and we can
represent the different scopes using parentheses:
\begin{itemize}
\item[(18)] \begin{itemize}
\item[a.] 
old (men and women)
\end{itemize}
\begin{itemize}
\item[b.] 
(old men) and women
\end{itemize}
\end{itemize}

One convenient way of representing this scope difference at a structural
level is by means of a \index{tree diagram}\textbf{tree diagram}, as shown in \href{\#tree-diagram}{(19b)}.
\hypertarget{tree-diagram}{}\begin{itemize}
\item[(19)] \begin{itemize}
\item[a.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-7.pdf}}}\end{itemize}
\begin{itemize}
\item[b.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-8.pdf}}}\end{itemize}
\end{itemize}

Note that linguistic trees grow upside down: the node labeled \docutilsrolecategory{s}
is the \index{root}\textbf{root} of the tree, while the \index{leaves}\textbf{leaves} of the tree are
labeled with the words.

In NLTK, you can easily produce trees like this yourself with the
following commands:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tree = nltk.bracket{\_}parse(\pysrcstring{'(NP (Adj old) (NP (N men) (Conj and) (N women)))'})
\pysrcprompt{>{}>{}> }tree.draw()                 }\end{alltt}

We can construct other examples of syntactic ambiguity
involving the coordinating conjunctions \docutilsroleexample{and} and \docutilsroleexample{or}, e.g.
\docutilsroleexample{Kim left or Dana arrived and everyone cheered}.
We can describe this ambiguity in terms of the relative
semantic \index{scope}\textbf{scope} of \docutilsroleexample{or} and \docutilsroleexample{and}.

For our third illustration of ambiguity, we look at
prepositional phrases.
Consider a sentence like: \docutilsroleexample{I saw the man with a telescope}.  Who
has the telescope?  To clarify what is going on here, consider the
following pair of sentences:
\begin{itemize}
\item[(20)] \begin{itemize}
\item[a.] 
The policeman saw a burglar \emph{with a gun}.
(not some other burglar)
\end{itemize}
\begin{itemize}
\item[b.] 
The policeman saw a burglar \emph{with a telescope}.
(not with his naked eye)
\end{itemize}
\end{itemize}

In both cases, there is a prepositional phrase introduced by
\docutilsroleexample{with}.  In the first case this phrase modifies the noun
\docutilsroleexample{burglar}, and in the second case it modifies the verb \docutilsroleexample{saw}.
We could again think of this in terms of scope: does the prepositional
phrase (\docutilsrolecategory{pp}) just have scope over the \docutilsrolecategory{np}
\docutilsroleexample{a burglar}, or does it have scope over
the whole verb phrase? As before, we can represent the difference in terms
of tree structure:
\hypertarget{burglar}{}\begin{itemize}
\item[(21)] \begin{itemize}
\item[a.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-9.pdf}}}\end{itemize}
\begin{itemize}
\item[b.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-10.pdf}}}\end{itemize}
\end{itemize}

{\noindent}
In \href{\#burglar}{(21b)}a, the \docutilsrolecategory{pp} attaches to the \docutilsrolecategory{np},
while in \href{\#burglar}{(21b)}b, the \docutilsrolecategory{pp} attaches to the \docutilsrolecategory{vp}.

We can generate these trees in Python as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }s1 = \pysrcstring{'(S (NP the policeman) (VP (V saw) (NP (NP the burglar) (PP with a gun))))'}
\pysrcprompt{>{}>{}> }s2 = \pysrcstring{'(S (NP the policeman) (VP (V saw) (NP the burglar) (PP with a telescope)))'}
\pysrcprompt{>{}>{}> }tree1 = nltk.bracket{\_}parse(s1)
\pysrcprompt{>{}>{}> }tree2 = nltk.bracket{\_}parse(s2)}\end{alltt}

We can discard the structure to get the list of \index{leaves}\textbf{leaves}, and
we can confirm that both trees have the same leaves (except for the last word).
We can also see that the trees have different \index{heights}\textbf{heights} (given by the
number of nodes in the longest branch of the tree, starting at \docutilsrolecategory{s}
and descending to the words):
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tree1.leaves()
\pysrcoutput{{[}'the', 'policeman', 'saw', 'the', 'burglar', 'with', 'a', 'gun'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }tree1.leaves(){[}:-1{]} == tree2.leaves(){[}:-1{]}
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }tree1.height() == tree2.height()
\pysrcoutput{False}}\end{alltt}

In general, how can we determine whether a prepositional phrase
modifies the preceding noun or verb? This problem is known as
\index{prepositional phrase attachment ambiguity}\textbf{prepositional phrase attachment ambiguity}.
The \index{Prepositional Phrase Attachment Corpus}\textbf{Prepositional Phrase Attachment Corpus} makes it
possible for us to study this question systematically.  The corpus is
derived from the IBM-Lancaster Treebank of Computer Manuals and from
the Penn Treebank, and distills out only the essential information
about \docutilsrolecategory{pp} attachment. Consider the sentence from the WSJ
in \href{\#ppattach-a}{(22a)}.  The corresponding line in the Prepositional Phrase
Attachment Corpus is shown in \href{\#ppattach-b}{(22b)}.
\begin{itemize}
\item[(22)] \hypertarget{ppattach-a}{}\begin{itemize}
\item[a.] 
Four of the five surviving workers have asbestos-related diseases,
including three with recently diagnosed cancer.
\end{itemize}
\hypertarget{ppattach-b}{}\begin{itemize}
\item[b.] \begin{quote}{\ttfamily \raggedright \noindent
\small
16~including~three~with~cancer~N
}\end{quote}
\end{itemize}
\end{itemize}

{\noindent}
That is, it includes an identifier for the original sentence, the
head of the relevant verb phrase (i.e., \docutilsroleexample{including}), the head of
the verb's \docutilsrolecategory{np} object (\docutilsroleexample{three}), the preposition
(\docutilsroleexample{with}), and the head noun within the prepositional phrase
(\docutilsroleexample{cancer}). Finally, it contains an ``attachment'' feature (\texttt{N} or
\texttt{V}) to indicate whether the prepositional phrase attaches to
(modifies) the noun phrase or the verb phrase.
Here are some further examples:
\hypertarget{ppattachments}{}\begin{itemize}
\item[(23)] \begin{quote}{\ttfamily \raggedright \noindent
\small
47830~allow~visits~between~families~N~\\
47830~allow~visits~on~peninsula~V~\\
42457~acquired~interest~in~firm~N~\\
42457~acquired~interest~in~1986~V
}\end{quote}
\end{itemize}

{\noindent}
The PP attachments in \href{\#ppattachments}{(23)} can also be made explicit by
using phrase groupings as in \href{\#phrase-groupings}{(24)}.
\hypertarget{phrase-groupings}{}\begin{itemize}
\item[(24)] \begin{quote}{\ttfamily \raggedright \noindent
\small
allow~(NP~visits~(PP~between~families))~\\
allow~(NP~visits)~(PP~on~peninsula)~\\
acquired~(NP~interest~(PP~in~firm))~\\
acquired~(NP~interest)~(PP~in~1986)
}\end{quote}
\end{itemize}

Observe in each case that the argument of the verb is either a single
complex expression \texttt{({\linebreak[0]}visits{\linebreak[0]} {\linebreak[0]}({\linebreak[0]}between{\linebreak[0]} {\linebreak[0]}families{\linebreak[0]}){\linebreak[0]})} or a pair of
simpler expressions \texttt{({\linebreak[0]}visits{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}({\linebreak[0]}on{\linebreak[0]} {\linebreak[0]}peninsula{\linebreak[0]})}.

We can access the Prepositional Phrase Attachment Corpus from NLTK as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.corpus.ppattach.tuples(\pysrcstring{'training'}){[}9{]}
\pysrcoutput{('16', 'including', 'three', 'with', 'cancer', 'N')}}\end{alltt}

If we go back to our first examples of \docutilsrolecategory{pp} attachment ambiguity,
it appears as though it is the \docutilsrolecategory{pp} itself (e.g., \docutilsroleexample{with a gun}
versus \docutilsroleexample{with a telescope}) that determines the attachment. However,
we can use this corpus to find examples where other factors come into play.
For example, it appears that the verb is the key factor in \href{\#ppattach-verb}{(25)}.
\hypertarget{ppattach-verb}{}\begin{itemize}
\item[(25)] \begin{quote}{\ttfamily \raggedright \noindent
\small
8582~received~offer~from~group~V~\\
19131~rejected~offer~from~group~N
}\end{quote}
\end{itemize}


%___________________________________________________________________________

\subsection{Constituency}
\label{constituency}

We claimed earlier that one of the motivations for building syntactic
structure was to help make explicit how a sentence says ``who did what
to whom''. Let's just focus for a while on the ``who'' part of this
story: in other words, how can syntax tell us what the subject of a
sentence is? At first, you might think this task is rather simple
— so simple indeed that we don't need to bother with syntax. In
a sentence such as \docutilsroleexample{The fierce dog bit the man}
we know that it is the dog that is doing the biting. So we could
say that the noun phrase immediately preceding the verb is the
subject of the sentence. And we might try to make this more explicit
in terms of sequences part-of-speech tags.  Let's try to come up with a simple
definition of \index{noun phrase}\textit{noun phrase}; we might start off with something
like this, based on our knowledge of noun phrase chunking (\href{\#chap-chunk}{Chapter 6}):
\begin{itemize}
\item[(26)] 
\docutilsrolecategory{dt jj* nn}
\end{itemize}

{\noindent}
We're using regular expression notation here in the form of
\docutilsrolecategory{jj*} to indicate a sequence of zero or more \docutilsrolecategory{jj}s. So this
is intended to say that a noun phrase can consist of a
determiner, possibly followed by some adjectives, followed by a
noun. Then we can go on to say that if we can find a sequence of
tagged words like this that precedes a word tagged as a verb, then
we've identified the subject. But now think about this sentence:
\begin{itemize}
\item[(27)] 
The child with a fierce dog bit the man.
\end{itemize}

{\noindent}
This time, it's the child that is doing the biting. But the tag
sequence preceding the verb is:
\begin{itemize}
\item[(28)] 
\docutilsrolecategory{dt nn in dt jj nn}
\end{itemize}

Our previous attempt at identifying the subject would have
incorrectly come up with \docutilsroleexample{the fierce dog} as the subject.
So our next hypothesis would have to be a bit more complex. For
example, we might say that the subject can be identified as any string
matching the following pattern before the verb:
\begin{itemize}
\item[(29)] 
\docutilsrolecategory{dt jj* nn (in dt jj* nn)*}
\end{itemize}

In other words, we need to find a noun phrase followed by zero or more
sequences consisting of a preposition followed by a noun phrase. Now
there are two unpleasant aspects to this proposed solution. The first
is esthetic: we are forced into repeating the sequence of tags (\docutilsrolecategory{dt
jj* nn}) that constituted our initial notion of noun phrase, and
our initial notion was in any case a drastic simplification. More
worrying, this approach still doesn't work! For consider the following
example:
\hypertarget{seagull}{}\begin{itemize}
\item[(30)] 
The seagull that attacked the child with the fierce dog bit the man.
\end{itemize}

{\noindent}
This time the seagull is the culprit, but it won't be detected as subject by our
attempt to match sequences of tags. So it seems that we need a
richer account of how words are \emph{grouped} together into patterns, and
a way of referring to these groupings at different points in the
sentence structure. This idea of grouping is often called
syntactic \index{constituency}\textbf{constituency}.

As we have just seen, a well-formed sentence of a language is more
than an arbitrary sequence of words from the language.  Certain kinds
of words usually go together.  For instance, determiners like \docutilsroleexample{the}
are typically followed by adjectives or nouns, but not by verbs.
Groups of words form intermediate structures called phrases or
\index{constituents}\textbf{constituents}.  These constituents can be identified using
standard syntactic tests, such as substitution, movement and
coordination.  For example, if a sequence of words can be replaced
with a pronoun, then that sequence is likely to be a constituent.
According to this test, we can infer that the italicized string in the
following example is a constituent, since it can be replaced by
\docutilsroleexample{they}:
\begin{itemize}
\item[(31)] \begin{itemize}
\item[a.] 
\emph{Ordinary daily multivitamin and mineral supplements} could
help adults with diabetes fight off some minor infections.
\end{itemize}
\begin{itemize}
\item[b.] 
\emph{They} could help adults with diabetes fight off some minor
infections.
\end{itemize}
\end{itemize}

In order to identify whether a phrase is the subject of a sentence, we
can use the construction called \index{Subject-Auxiliary Inversion}\textbf{Subject-Auxiliary Inversion} in
English. This construction allows us to form so-called Yes-No
Questions. That is, corresponding to the statement in \href{\#have1}{(32a)}, we have
the question in \href{\#have2}{(32b)}:
\begin{itemize}
\item[(32)] \hypertarget{have1}{}\begin{itemize}
\item[a.] 
All the cakes have been eaten.
\end{itemize}
\hypertarget{have2}{}\begin{itemize}
\item[b.] 
Have \emph{all the cakes} been eaten?
\end{itemize}
\end{itemize}

Roughly speaking, if a sentence already contains an auxiliary verb,
such as \docutilsroleexample{has} in \href{\#have1}{(32a)}, then we can turn it into a Yes-No
Question by moving the auxiliary verb 'over' the subject noun phrase
to the front of the sentence. If there is no auxiliary in the
statement, then we insert the appropriate form of \docutilsroleexample{do} as the
fronted auxiliary and replace the tensed main verb by its base form:
\begin{itemize}
\item[(33)] \begin{itemize}
\item[a.] 
The fierce dog bit the man.
\end{itemize}
\begin{itemize}
\item[b.] 
Did \emph{the fierce dog} bite the man?
\end{itemize}
\end{itemize}

As we would hope, this test also confirms our earlier claim about the
subject constituent of \href{\#seagull}{(30)}:
\begin{itemize}
\item[(34)] 
Did \emph{the seagull that attacked the child with the fierce dog} bite
the man?
\end{itemize}

To sum up then, we have seen that the notion of constituent brings a
number of benefits. By having a constituent labeled \docutilsrolecategory{noun phrase},
we can provide a unified statement of the classes of word that
constitute that phrase, and reuse this statement in describing noun
phrases wherever they occur in the sentence. Second, we can use the
notion of a noun phrase in defining the subject of sentence, which in
turn is a crucial ingredient in determining the ``who does what to
whom'' aspect of meaning.


%___________________________________________________________________________

\subsection{More on Trees}
\label{more-on-trees}

A tree is a set of connected nodes, each of which is labeled with a
category.  It common to use a 'family' metaphor to talk about the
relationships of nodes in a tree: for example, \docutilsrolecategory{s} is the
\index{parent}\textbf{parent} of \docutilsrolecategory{vp}; conversely \docutilsrolecategory{vp} is a \index{daughter}\textbf{daughter} (or
\index{child}\textbf{child}) of \docutilsrolecategory{s}.  Also, since \docutilsrolecategory{np} and \docutilsrolecategory{vp} are both
daughters of \docutilsrolecategory{s}, they are also \index{sisters}\textbf{sisters}.
Here is an example of a tree:
\begin{itemize}
\item[(35)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-11.pdf}}}\end{itemize}

Although it is helpful to represent trees in a graphical format, for
computational purposes we usually need a more text-oriented
representation.  We will use the same format as the Penn Treebank,
a combination of brackets and labels:
\begin{alltt}\small\textbf{(S
   (NP Lee)
   (VP
      (V saw)
      (NP
         (Det the)
         (N dog))))}\end{alltt}

Here, the node value is a constituent type (e.g., \docutilsrolecategory{np} or \docutilsrolecategory{vp}),
and the children encode the hierarchical contents of the tree.

Although we will focus on syntactic trees, trees can be used to encode
\docutilsroleemphasis{any} homogeneous hierarchical structure that spans a sequence
of linguistic forms (e.g. morphological structure, discourse structure).
In the general case, leaves and node values do not have to be strings.

In NLTK, trees are created with the \texttt{Tree} constructor, which takes a
node value and a list of zero or more children.  Here's a couple of
simple trees:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tree1 = nltk.Tree(\pysrcstring{'NP'}, {[}\pysrcstring{'John'}{]})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} tree1
\pysrcoutput{(NP John)}
\pysrcoutput{}\pysrcprompt{>{}>{}> }tree2 = nltk.Tree(\pysrcstring{'NP'}, {[}\pysrcstring{'the'}, \pysrcstring{'man'}{]})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} tree2
\pysrcoutput{(NP the man)}}\end{alltt}

{\noindent}
We can incorporate these into successively larger trees as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tree3 = nltk.Tree(\pysrcstring{'VP'}, {[}\pysrcstring{'saw'}, tree2{]})
\pysrcprompt{>{}>{}> }tree4 = nltk.Tree(\pysrcstring{'S'}, {[}tree1, tree3{]})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} tree4
\pysrcoutput{(S (NP John) (VP saw (NP the man)))}}\end{alltt}

{\noindent}
Here are some of the methods available for tree objects:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} tree4{[}1{]}
\pysrcoutput{(VP saw (NP the man))}
\pysrcoutput{}\pysrcprompt{>{}>{}> }tree4{[}1{]}.node
\pysrcoutput{'VP'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }tree4.leaves()
\pysrcoutput{{[}'John', 'saw', 'the', 'man'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }tree4{[}1,1,1{]}
\pysrcoutput{'man'}}\end{alltt}

The printed representation for complex trees can be difficult to read.
In these cases, the \texttt{draw} method can be very useful.
It opens a new window, containing a graphical representation
of the tree.  The tree display window allows you to zoom in and out;
to collapse and expand subtrees; and to print the graphical
representation to a postscript file (for inclusion in a document).
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tree3.draw()                           }\end{alltt}

\scalebox{0.360000}{\includegraphics{../images/parse_draw.png}}


%___________________________________________________________________________

\subsection{Treebanks (notes)}
\label{treebanks-notes}

The \texttt{corpus} module defines the \texttt{treebank} corpus reader,
which contains a 10{\%} sample of the Penn Treebank corpus.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.corpus.treebank.parsed{\_}sents(\pysrcstring{'wsj{\_}0001.mrg'}){[}0{]}
\pysrcoutput{(S}
\pysrcoutput{  (NP-SBJ}
\pysrcoutput{    (NP (NNP Pierre) (NNP Vinken))}
\pysrcoutput{    (, ,)}
\pysrcoutput{    (ADJP (NP (CD 61) (NNS years)) (JJ old))}
\pysrcoutput{    (, ,))}
\pysrcoutput{  (VP}
\pysrcoutput{    (MD will)}
\pysrcoutput{    (VP}
\pysrcoutput{      (VB join)}
\pysrcoutput{      (NP (DT the) (NN board))}
\pysrcoutput{      (PP-CLR}
\pysrcoutput{        (IN as)}
\pysrcoutput{        (NP (DT a) (JJ nonexecutive) (NN director)))}
\pysrcoutput{      (NP-TMP (NNP Nov.) (CD 29))))}
\pysrcoutput{  (. .))}}\end{alltt}

\href{\#indent-tree}{Listing 7.1} prints a tree object using whitespace formatting.
\hypertarget{indent-tree}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{indent{\_}tree}(t, level=0, first=False, width=8):
    \pysrckeyword{if} \pysrckeyword{not} first:
        \pysrckeyword{print} \pysrcstring{' '}*(width+1)*level,
    try:
        \pysrckeyword{print} \pysrcstring{"{\%}-*s"} {\%} (width, t.node),
        indent{\_}tree(t{[}0{]}, level+1, first=True)
        \pysrckeyword{for} child \pysrckeyword{in} t{[}1:{]}:
            indent{\_}tree(child, level+1, first=False)
    \pysrckeyword{except} AttributeError:
        \pysrckeyword{print} t}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }t = nltk.corpus.treebank.parsed{\_}sents(\pysrcstring{'wsj{\_}0001.mrg'}){[}0{]}
\pysrcprompt{>{}>{}> }indent{\_}tree(t)
\pysrcoutput{ S        NP-SBJ   NP       NNP      Pierre}
\pysrcoutput{                            NNP      Vinken}
\pysrcoutput{                   ,        ,}
\pysrcoutput{                   ADJP     NP       CD       61}
\pysrcoutput{                                     NNS      years}
\pysrcoutput{                            JJ       old}
\pysrcoutput{                   ,        ,}
\pysrcoutput{          VP       MD       will}
\pysrcoutput{                   VP       VB       join}
\pysrcoutput{                            NP       DT       the}
\pysrcoutput{                                     NN       board}
\pysrcoutput{                            PP-CLR   IN       as}
\pysrcoutput{                                     NP       DT       a}
\pysrcoutput{                                              JJ       nonexecutive}
\pysrcoutput{                                              NN       director}
\pysrcoutput{                            NP-TMP   NNP      Nov.}
\pysrcoutput{                                     CD       29}
\pysrcoutput{          .        .}}\end{alltt}
\caption{}
\vspace{1ex}
\hrule
\end{pylisting}

NLTK also includes a sample from the \emph{Sinica Treebank Corpus},
consisting of 10,000 parsed sentences drawn from the
\emph{Academia Sinica Balanced Corpus of Modern Chinese}.
Here is a code fragment to read and display one of the trees in this corpus.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.corpus.sinica{\_}treebank.parsed{\_}sents(){[}3450{]}.draw()               }\end{alltt}
\hypertarget{sinica-tree}{}\begin{itemize}
\item[(36)] \scalebox{0.360000}{\includegraphics{../images/sinica-tree.png}}\end{itemize}

Note that we can read tagged text from a Treebank corpus, using the \texttt{tagged{\linebreak[0]}({\linebreak[0]})} method:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.corpus.treebank.tagged{\_}sents(\pysrcstring{'wsj{\_}0001.mrg'}){[}0{]}
\pysrcoutput{{[}('Pierre', 'NNP'), ('Vinken', 'NNP'), (',', ','), ('61', 'CD'), ('years', 'NNS'),}
\pysrcoutput{('old', 'JJ'), (',', ','), ('will', 'MD'), ('join', 'VB'), ('the', 'DT'),}
\pysrcoutput{('board', 'NN'), ('as', 'IN'), ('a', 'DT'), ('nonexecutive', 'JJ'),}
\pysrcoutput{('director', 'NN'), ('Nov.', 'NNP'), ('29', 'CD'), ('.', '.'){]}}}\end{alltt}


%___________________________________________________________________________

\subsection{Exercises}
\label{id68}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Can you come up with grammatical sentences that have probably never
been uttered before?  (Take turns with a partner.)  What does this tell you
about human language?

\item {} 
☼ Recall Strunk and White's prohibition against sentence-initial
\docutilsroleexample{however} used to mean ``although''.
Do a web search for \docutilsroleexample{however} used at the start of the sentence.
How widely used is this construction?

\item {} 
☼ Consider the sentence \titlereference{Kim arrived or Dana left and everyone cheered}.
Write down the parenthesized forms to show the relative scope of \docutilsroleexample{and}
and \docutilsroleexample{or}.  Generate tree structures corresponding to both of these interpretations.

\item {} 
☼ The \texttt{Tree} class implements a variety of other useful methods.
See the \texttt{Tree} help documentation for more details, i.e. import
the Tree class and then type \texttt{help{\linebreak[0]}({\linebreak[0]}Tree{\linebreak[0]})}.

\item {} 
☼ \textbf{Building trees:}
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Write code to produce two trees, one for each reading of the phrase
\docutilsroleexample{old men and women}

\item {} 
Encode any of the trees presented in this chapter as a labeled
bracketing and use \texttt{nltk{\linebreak[0]}.{\linebreak[0]}bracket{\_}parse{\linebreak[0]}({\linebreak[0]})} to check that it is well-formed.
Now use \texttt{draw{\linebreak[0]}({\linebreak[0]})} to display the tree.

\item {} 
As in (a) above, draw a tree for \docutilsroleexample{The woman saw a man last Thursday}.

\end{list}

\item {} 
☼ Write a recursive function to traverse a tree and return the
depth of the tree, such that a tree with a single node would have
depth zero.  (Hint: the depth of a subtree is the maximum depth
of its children, plus one.)

\item {} 
☼ Analyze the A.A. Milne sentence about Piglet, by underlining all
of the sentences it contains then replacing these with \docutilsrolecategory{s}
(e.g. the first sentence becomes \docutilsrolecategory{s} \titlereference{when}:lx` \docutilsrolecategory{s}).
Draw a tree structure for this ``compressed'' sentence.  What are
the main syntactic constructions used for building such a long
sentence?

\item {} 
◑ To compare multiple trees in a single window, we can use the
\texttt{draw{\_}trees{\linebreak[0]}({\linebreak[0]})} method.  Define some trees and try it out:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk.draw.tree \pysrckeyword{import} draw{\_}trees
\pysrcprompt{>{}>{}> }draw{\_}trees(tree1, tree2, tree3)                    }\end{alltt}

\item {} 
◑ Using tree positions, list the subjects of the first 100
sentences in the Penn treebank; to make the results easier to view,
limit the extracted subjects to subtrees whose height is 2.

\item {} 
◑ Inspect the Prepositional Phrase Attachment Corpus
and try to suggest some factors that influence \docutilsrolecategory{pp} attachment.

\item {} 
◑ In this section we claimed that there are linguistic regularities
that cannot be described simply in terms of n-grams.
Consider the following sentence, particularly the position of the phrase
\docutilsroleexample{in his turn}.  Does this illustrate a problem for an approach based
on n-grams?
\begin{quote}

\titlereference{What was more, the in his turn somewhat youngish Nikolay Parfenovich
also turned out to be the only person in the entire world to acquire a
sincere liking to our ``discriminated-against'' public procurator.}
(Dostoevsky: The Brothers Karamazov)
\end{quote}

\item {} 
◑ Write a recursive function that produces a nested bracketing for
a tree, leaving out the leaf nodes, and displaying the non-terminal
labels after their subtrees.  So the above example about Pierre
Vinken would produce:
\texttt{{[}{\linebreak[0]}{[}{\linebreak[0]}{[}{\linebreak[0]}NNP{\linebreak[0]} {\linebreak[0]}NNP{\linebreak[0]}{]}{\linebreak[0]}NP{\linebreak[0]} {\linebreak[0]},{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}ADJP{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}CD{\linebreak[0]} {\linebreak[0]}NNS{\linebreak[0]}{]}{\linebreak[0]}NP{\linebreak[0]} {\linebreak[0]}JJ{\linebreak[0]}{]}{\linebreak[0]}ADJP{\linebreak[0]} {\linebreak[0]},{\linebreak[0]}{]}{\linebreak[0]}NP{\linebreak[0]}-{\linebreak[0]}SBJ{\linebreak[0]} {\linebreak[0]}MD{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}VB{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}DT{\linebreak[0]} {\linebreak[0]}NN{\linebreak[0]}{]}{\linebreak[0]}NP{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}IN{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}DT{\linebreak[0]} {\linebreak[0]}JJ{\linebreak[0]} {\linebreak[0]}NN{\linebreak[0]}{]}{\linebreak[0]}NP{\linebreak[0]}{]}{\linebreak[0]}PP{\linebreak[0]}-{\linebreak[0]}CLR{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}NNP{\linebreak[0]} {\linebreak[0]}CD{\linebreak[0]}{]}{\linebreak[0]}NP{\linebreak[0]}-{\linebreak[0]}TMP{\linebreak[0]}{]}{\linebreak[0]}VP{\linebreak[0]} {\linebreak[0]}.{\linebreak[0]}{]}{\linebreak[0]}S}
Consecutive categories should be separated by space.

\end{list}
% recurse over tree to look for coordinate constructions (cf 4th
% example in chapter 1.1); (possible extension: callback function for Tree.subtrees()) 
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
◑ Download several electronic books from Project Gutenberg.
Write a program to scan these texts for any extremely long sentences.
What is the longest sentence you can find?  What syntactic construction(s)
are responsible for such long sentences?

\item {} 
★ One common way of defining the subject of a sentence \docutilsrolecategory{s} in
English is as \emph{the noun phrase that is the daughter of} \docutilsrolecategory{s} \emph{and
the sister of} \docutilsrolecategory{vp}.   Write a function that takes the tree for
a sentence and returns the subtree corresponding to the subject of the
sentence.  What should it do if the root node of the tree passed to
this function is not \docutilsrolecategory{s}, or it lacks a subject?

\end{list}


%___________________________________________________________________________

\section{Context Free Grammar}
\label{context-free-grammar}

As we have seen, languages are infinite — there is no principled
upper-bound on the length of a sentence.  Nevertheless, we would like
to write (finite) programs that can process well-formed sentences.  It turns
out that we can characterize what we mean by well-formedness using a
grammar.  The way that finite grammars are able to describe an
infinite set uses \index{recursion}\textbf{recursion}.  (We already came across this idea
when we looked at regular expressions: the finite expression \texttt{a{\linebreak[0]}+} is
able to describe the infinite set \texttt{{\{}{\linebreak[0]}a{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}aa{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}aaa{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}aaaa{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}{\}}}).  Apart
from their compactness, grammars usually capture important structural
and distributional properties of the language, and can be used to map
between sequences of words and abstract representations of meaning.
Even if we were to impose an upper bound on sentence length to ensure
the language was finite, we would probably still want to come up with
a compact representation in the form of a grammar.

A \index{grammar}\textbf{grammar} is a formal system that specifies which sequences of
words are well-formed in the language, and that provides one or more
phrase structures for well-formed sequences.  We will be looking at
\index{context-free grammar}\textbf{context-free grammar} (CFG), which is a collection of
\index{productions}\textbf{productions} of the form \docutilsrolecategory{s} → \docutilsrolecategory{np vp}.  This says
that a constituent \docutilsrolecategory{s} can consist of sub-constituents \docutilsrolecategory{np}
and \docutilsrolecategory{vp}. Similarly, the production \docutilsrolecategory{v} → \texttt{\pysrcstring{'{\linebreak[0]}saw{\linebreak[0]}'} {\linebreak[0]}|{\linebreak[0]} {\linebreak[0]}`{\linebreak[0]}`\pysrcstring{'{\linebreak[0]}walked{\linebreak[0]}'}}
means that the constituent \docutilsrolecategory{v} can consist of the string
\docutilsroleexample{saw} or \docutilsroleexample{walked}.
For a phrase structure tree to be well-formed relative to
a grammar, each non-terminal node and its children must correspond to
a production in the grammar.


%___________________________________________________________________________

\subsection{A Simple Grammar}
\label{a-simple-grammar}

Let's start off by looking at a simple context-free grammar.
By convention, the left-hand-side of the first production is
the \index{start-symbol}\textbf{start-symbol} of the grammar, and all well-formed trees
must have this symbol as their root label.
\hypertarget{grammar1}{}\begin{itemize}
\item[(37)] 
\begin{lineblock}{0em}
\item[] S → NP VP
\item[] NP → Det N {\textbar} Det N PP
\item[] VP → V {\textbar} V NP {\textbar} V NP PP
\item[] PP → P NP
\item[] 
\item[] Det → 'the' {\textbar} 'a'
\item[] N → 'man' {\textbar} 'park' {\textbar} 'dog' {\textbar} 'telescope'
\item[] V → 'saw' {\textbar} 'walked'
\item[] P → 'in' {\textbar} 'with'
\end{lineblock}
\end{itemize}

This grammar contains productions involving various syntactic categories,
as laid out in \href{\#syncat}{Table 7.1}.
\hypertarget{syncat}{}
\leavevmode
\begin{longtable}[c]{|p{0.086\locallinewidth}|p{0.249\locallinewidth}|p{0.272\locallinewidth}|}
\hline
\textbf{
Symbol
} & \textbf{
Meaning
} & \textbf{
Example
} \\
\hline
\endhead

S
 & 
sentence
 & 
\docutilsroleexample{the man walked}
 \\
\hline

NP
 & 
noun phrase
 & 
\docutilsroleexample{a dog}
 \\
\hline

VP
 & 
verb phrase
 & 
\docutilsroleexample{saw a park}
 \\
\hline

PP
 & 
prepositional phrase
 & 
\docutilsroleexample{with a telescope}
 \\
\hline

...
 & 
...
 & 
...
 \\
\hline

Det
 & 
determiner
 & 
\docutilsroleexample{the}
 \\
\hline

N
 & 
noun
 & 
\docutilsroleexample{dog}
 \\
\hline

V
 & 
verb
 & 
\docutilsroleexample{walked}
 \\
\hline

P
 & 
preposition
 & 
\docutilsroleexample{in}
 \\
\hline
\caption{
Syntactic Categories
}\end{longtable}

In our following discussion of grammar, we will use the following terminology.
The grammar consists of productions, where each production involves a
single \index{non-terminal}\textbf{non-terminal} (e.g. \docutilsrolecategory{s}, \docutilsrolecategory{np}), an arrow, and one
or more non-terminals and \index{terminals}\textbf{terminals} (e.g. \docutilsroleexample{walked}).
The productions are often divided into two main groups.
The \index{grammatical productions}\textbf{grammatical productions} are those without a terminal on
the right hand side.  The \index{lexical productions}\textbf{lexical productions} are those having
a terminal on the right hand side.
A special case of non-terminals are the \index{pre-terminals}\textbf{pre-terminals}, which
appear on the left-hand side of lexical productions.
We will say that a grammar \index{licenses}\textbf{licenses} a tree if each non-terminal
\docutilsrolecategory{x} with children \docutilsrolecategory{y}$_{1}$ ... \docutilsrolecategory{y}$_{n}$
corresponds to a production in the grammar of the form:
\docutilsrolecategory{x} → \docutilsrolecategory{y}$_{1}$ ... \docutilsrolecategory{y}$_{n}$.

In order to get started with developing simple grammars of your own, you
will probably find it convenient to play with the recursive descent
parser demo, \texttt{nltk{\linebreak[0]}.{\linebreak[0]}draw{\linebreak[0]}.{\linebreak[0]}rdparser{\linebreak[0]}.{\linebreak[0]}demo{\linebreak[0]}({\linebreak[0]})}.
The demo opens a window that displays a list of grammar productions in the
left hand pane and the current parse diagram in the central pane:

\scalebox{0.514286}{\includegraphics{../images/parse_rdparsewindow.png}}

The demo comes with the grammar in \href{\#grammar1}{(37)} already loaded. We will
discuss the parsing algorithm in greater detail below, but for the
time being you can get an idea of how it works by using the \emph{autostep} button.
If we parse the string \titlereference{The dog saw a man in the park} using
the grammar in \href{\#grammar1}{(37)}, we end up with two trees:
\begin{itemize}
\item[(38)] \begin{itemize}
\item[a.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-12.pdf}}}\end{itemize}
\begin{itemize}
\item[b.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-13.pdf}}}\end{itemize}
\end{itemize}

Since our grammar licenses two trees for this sentence, the sentence is
said to be \index{structurally ambiguous}\textbf{structurally ambiguous}.  The ambiguity in question is called
a \index{prepositional phrase attachment ambiguity}\textit{prepositional phrase attachment ambiguity}, as we saw earlier in this chapter.
As you may recall, it is an ambiguity about attachment since the
\docutilsrolecategory{pp} \docutilsroleexample{in the park} needs to be attached to one of two places
in the tree: either as a daughter of \docutilsrolecategory{VP} or else as a daughter of
\docutilsrolecategory{np}.
When the \docutilsrolecategory{pp} is attached to \docutilsrolecategory{vp}, the seeing event happened
in the park.  However, if the \docutilsrolecategory{pp} is attached to \docutilsrolecategory{np},
then the man was in the park, and the agent of the seeing (the dog)
might have been sitting on the balcony of an apartment overlooking the park.
As we will see, dealing with ambiguity is a key challenge in parsing.


%___________________________________________________________________________

\subsection{Recursion in Syntactic Structure}
\label{recursion-in-syntactic-structure}

Observe that sentences can be nested within sentences, with no limit
to the depth:
\begin{itemize}
\item[(39)] \begin{itemize}
\item[a.] 
Jodie won the 100m freestyle
\end{itemize}
\begin{itemize}
\item[b.] 
``The Age'' reported that Jodie won the 100m freestyle
\end{itemize}
\begin{itemize}
\item[c.] 
Sandy said ``The Age'' reported that Jodie won the 100m freestyle
\end{itemize}
\begin{itemize}
\item[d.] 
I think Sandy said ``The Age'' reported that Jodie won the 100m freestyle
\end{itemize}
\end{itemize}

{\noindent}
This nesting is explained in terms of \index{recursion}\textbf{recursion}. A grammar is
said to be \index{recursive}\textbf{recursive} if a category occurring on the left hand
side of a production (such as \docutilsrolecategory{s} in this case) also appears on
the right hand side of a production. If this dual occurrence takes
place in \emph{one and the same production}, then we have \index{direct
recursion}\textbf{direct
recursion}; otherwise we have \index{indirect recursion}\textbf{indirect recursion}. There is no
recursion in \href{\#grammar1}{(37)}. However, the grammar in \href{\#grammar2}{(40)} illustrates both kinds of
recursive production:
\hypertarget{grammar2}{}\begin{itemize}
\item[(40)] \begin{quote}{\ttfamily \raggedright \noindent
\small
S~~→~NP~VP~\\
NP~→~Det~Nom~|~Det~Nom~PP~|~PropN~\\
Nom~→~Adj~Nom~|~N~\\
VP~→~V~|~V~NP~|~V~NP~PP~|~V~S~\\
PP~→~P~NP~\\
~\\
PropN~→~'John'~|~'Mary'~\\
Det~→~'the'~|~'a'~\\
N~→~'man'~|~'woman'~|~'park'~|~'dog'~|~'lead'~|~'telescope'~|~'butterfly'~\\
Adj~~→~'fierce'~|~'black'~|~~'big'~|~'European'~\\
V~→~'saw'~|~'chased'~|~'barked'~~|~'disappeared'~|~'said'~|~'reported'~\\
P~→~'in'~|~'with'
}\end{quote}
\end{itemize}

Notice that the production \docutilsrolecategory{Nom} → \docutilsrolecategory{Adj Nom} (where
\docutilsrolecategory{Nom} is the category of nominals) involves direct
recursion on the category \docutilsrolecategory{Nom}, whereas indirect recursion on \docutilsrolecategory{s}
arises from the combination of two productions, namely \docutilsrolecategory{s} →
\docutilsrolecategory{np vp} and \docutilsrolecategory{vp} → \docutilsrolecategory{v s}.

To see how recursion is handled in this grammar, consider the following
trees.  Example \href{\#nested-nominals}{nested-nominals} involves nested nominal phrases,
while \href{\#nested-sentences}{nested-sentences} contains nested sentences.
\begin{itemize}
\item[(41)] \begin{itemize}
\item[a.] \hypertarget{nested-nominals}{}\raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-14.pdf}}}\end{itemize}
\begin{itemize}
\item[b.] \hypertarget{nested-sentences}{}\raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-15.pdf}}}\end{itemize}
\end{itemize}

If you did the exercises for the last section, you will have noticed
that the recursive descent parser fails to deal properly with the
following production: \docutilsrolecategory{np} → \docutilsrolecategory{np pp}.
From a linguistic point of view, this production is perfectly respectable,
and will allow us to derive trees like this:
\begin{itemize}
\item[(42)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-16.pdf}}}\end{itemize}

More schematically, the trees for these compound noun phrases will be
of the following shape:
\hypertarget{leftrec}{}\begin{itemize}
\item[(43)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-17.pdf}}}\end{itemize}

The structure in \href{\#leftrec}{(43)} is called a \index{left recursive}\textbf{left recursive} structure.
These occur frequently in analyses of English, and
the failure of recursive descent parsers to deal adequately with left
recursion means that we will need to find alternative approaches.


%___________________________________________________________________________

\subsection{Heads, Complements and Modifiers}
\label{heads-complements-and-modifiers}

Let us take a closer look at verbs.
The grammar \href{\#grammar2}{(40)} correctly generates examples like \href{\#subcat1}{(44d)},
corresponding to the four productions with \docutilsrolecategory{vp} on the left hand side:
\hypertarget{subcat1}{}\begin{itemize}
\item[(44)] \begin{itemize}
\item[a.] 
The woman gave the telescope to the dog
\end{itemize}
\begin{itemize}
\item[b.] 
The woman saw a man
\end{itemize}
\begin{itemize}
\item[c.] 
A man said that the woman disappeared
\end{itemize}
\begin{itemize}
\item[d.] 
The dog barked
\end{itemize}
\end{itemize}

That is, \docutilsroleexample{gave} can occur with a following \docutilsrolecategory{np} and \docutilsrolecategory{pp};
\docutilsroleexample{saw} can occur with a following \docutilsrolecategory{np};
\docutilsroleexample{said} can occur with a following \docutilsrolecategory{s};
and \docutilsroleexample{barked} can occur with no following phrase.
In these cases, \docutilsrolecategory{np}, \docutilsrolecategory{pp} and \docutilsrolecategory{s} are called \index{complements}\textbf{complements}
of the respective verbs, and the verbs themselves are called
\index{heads}\textbf{heads} of the verb phrase.

However, there are fairly strong constraints on what verbs can occur
with what complements. Thus, we would like our grammars to mark the
following examples as ungrammatical\footnotemark[1]:
\hypertarget{subcat2}{}\begin{itemize}
\item[(45)] \begin{itemize}
\item[a.] 
*The woman disappeared the telescope to the dog
\end{itemize}
\begin{itemize}
\item[b.] 
*The dog barked a man
\end{itemize}
\begin{itemize}
\item[c.] 
*A man gave that the woman disappeared
\end{itemize}
\begin{itemize}
\item[d.] 
*A man said
\end{itemize}
\end{itemize}
\footnotetext[1]{
It should be borne in mind that it is possible to create
examples that involve 'non-standard' but interpretable
combinations of verbs and complements. Thus, we can, at a stretch,
interpret \docutilsroleexample{the man disappeared the dog} as meaning that the man
made the dog disappear. We will ignore such examples here.
}

How can we ensure that our grammar correctly excludes the
ungrammatical examples in \href{\#subcat2}{(45d)}?  We need some way of constraining
grammar productions which expand \docutilsrolecategory{vp} so that verbs \emph{only} co-occur
with their correct complements. We do this by dividing the class of
verbs into \index{subcategories}\textbf{subcategories}, each of which is associated with a
different set of complements. For example, \index{transitive verbs}\textbf{transitive verbs} such
as \docutilsroleexample{saw}, \docutilsroleexample{kissed} and \docutilsroleexample{hit} require a following \docutilsrolecategory{np}
object complement. Borrowing from the terminology of chemistry, we
sometimes refer to the \index{valency}\textbf{valency} of a verb, that is, its capacity
to combine with a sequence of arguments and thereby compose a verb
phrase.

Let's introduce a new category label for such verbs, namely
\docutilsrolecategory{tv} (for Transitive Verb), and use it in the following productions:
\begin{itemize}
\item[(46)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{vp}~→~\docutilsrolecategory{tv~np}~\\
\docutilsrolecategory{tv}~→~'saw'~|~'kissed'~|~'hit'
}\end{quote}
\end{itemize}

Now \docutilsroleexample{*the dog barked the man} is excluded since we haven't listed
\docutilsroleexample{barked} as a \docutilsrolecategory{V{\_}tr}, but \docutilsroleexample{the woman saw a man} is still allowed.
\href{\#verbcat}{Table 7.2} provides more examples of labels for verb subcategories.
\hypertarget{verbcat}{}
\leavevmode
\begin{longtable}[c]{|p{0.086\locallinewidth}|p{0.249\locallinewidth}|p{0.296\locallinewidth}|}
\hline
\textbf{
Symbol
} & \textbf{
Meaning
} & \textbf{
Example
} \\
\hline
\endhead

IV
 & 
intransitive verb
 & 
\emph{barked}
 \\
\hline

TV
 & 
transitive verb
 & 
\emph{saw a man}
 \\
\hline

DatV
 & 
dative verb
 & 
\emph{gave a dog to a man}
 \\
\hline

SV
 & 
sentential verb
 & 
\emph{said that a dog barked}
 \\
\hline
\caption{
Verb Subcategories
}\end{longtable}

The revised grammar for \docutilsrolecategory{vp} will now look like this:
\hypertarget{subcat3}{}\begin{itemize}
\item[(47)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{vp}~→~\docutilsrolecategory{datv~np~pp}~\\
\docutilsrolecategory{vp}~→~\docutilsrolecategory{tv~np}~\\
\docutilsrolecategory{vp}~→~\docutilsrolecategory{sv~s}~\\
\docutilsrolecategory{vp}~→~\docutilsrolecategory{iv}~\\
~\\
\docutilsrolecategory{datv}~→~'gave'~|~'donated'~|~'presented'~\\
\docutilsrolecategory{tv}~→~'saw'~|~'kissed'~|~'hit'~|~'sang'~\\
\docutilsrolecategory{sv}~→~'said'~|~'knew'~|~'alleged'~\\
\docutilsrolecategory{iv}~→~'barked'~|~'disappeared'~|~'elapsed'~|~'sang'
}\end{quote}
\end{itemize}

Notice that according to \href{\#subcat3}{(47)}, a given lexical item can belong to more
than one subcategory. For example, \docutilsroleexample{sang} can occur both with and
without a following \docutilsrolecategory{np} complement.


%___________________________________________________________________________

\subsection{Dependency Grammar}
\label{dependency-grammar}

Although we concentrate on phrase structure grammars in this chapter,
we should mention an alternative approach, namely \index{dependency
grammar}\textbf{dependency
grammar}. Rather than taking starting from the grouping of words
into constituents, dependency grammar takes as basic the notion that
one word can be dependent on another (namely, its head). The root of a
sentence is usually taken to be the main verb, and every other word is
either dependent on the root, or connects to it through a path of
dependencies. \href{\#depgraph0}{Figure (48)} illustrates a dependency graph, where
the head of the arrow points to the head of a dependency.
\hypertarget{depgraph0}{}\begin{itemize}
\item[(48)] \scalebox{0.154286}{\includegraphics{../images/depgraph0.png}}\end{itemize}

As you will see, the arcs in \href{\#depgraph0}{Figure (48)} are labeled with the
particular dependency relation that holds between a dependent and its
head. For example, \docutilsroleexample{Esso} bears the subject relation to \docutilsroleexample{said}
(which is the head of the whole sentence), and \docutilsroleexample{Tuesday} bears a
verbal modifier (\docutilsrolecategory{vmod}) relation to \docutilsroleexample{started}.

An alternative way of representing the dependency relationships is illustrated
in the \href{\#depgraph1}{tree (49)},
where dependents are shown as daughters of their heads.
\hypertarget{depgraph1}{}\begin{itemize}
\item[(49)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-18.pdf}}}\end{itemize}

One format for encoding dependency information places each word on a
line, followed by its part-of-speech tag, the index of its head, and
the label of the dependency relation (cf. \cite{nivre2006mp}). The index
of a word is implicitly given by the ordering of the lines (with 1 as the
first index). This is illustrated in the following code snippet:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk{\_}contrib.dependency \pysrckeyword{import} DepGraph
\pysrcprompt{>{}>{}> }dg = DepGraph().read(\pysrcstring{"{}"{}"Esso    NNP     2       SUB}
\pysrcmore{... }\pysrcstring{said    VBD     0       ROOT}
\pysrcmore{... }\pysrcstring{the     DT      5       NMOD}
\pysrcmore{... }\pysrcstring{Whiting NNP     5       NMOD}
\pysrcmore{... }\pysrcstring{field   NN      6       SUB}
\pysrcmore{... }\pysrcstring{started VBD     2       VMOD}
\pysrcmore{... }\pysrcstring{production      NN      6       OBJ}
\pysrcmore{... }\pysrcstring{Tuesday NNP     6       VMOD"{}"{}"})}\end{alltt}

As you will see, this format also adopts the convention that the head
of the sentence is dependent on an empty node, indexed as 0. We can
use the \texttt{deptree{\linebreak[0]}({\linebreak[0]})} method of a \texttt{DepGraph{\linebreak[0]}({\linebreak[0]})} object to build an NLTK
tree like that illustrated earlier in \href{\#depgraph1}{(49)}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tree = dg.deptree()
\pysrcprompt{>{}>{}> }tree.draw()                                 }\end{alltt}


%___________________________________________________________________________

\subsection{Formalizing Context Free Grammars}
\label{formalizing-context-free-grammars}

We have seen that a CFG contains terminal and nonterminal symbols, and
productions that dictate how constituents are expanded into other
constituents and words. In this section, we provide some formal definitions.

A CFG is a 4-tuple 〈\docutilsrolemath{N}, Σ, \docutilsrolemath{P}, \docutilsrolemath{S}〉, where:
\begin{itemize}
\item {} 
Σ is a set of \emph{terminal} symbols (e.g., lexical items);

\item {} 
\docutilsrolemath{N} is a set of \emph{non-terminal} symbols (the category labels);

\item {} 
\docutilsrolemath{P} is a set of \emph{productions} of the form \docutilsrolemath{A} → α, where
\begin{itemize}
\item {} 
\docutilsrolemath{A} is a non-terminal, and

\item {} 
α is a string of symbols from (\docutilsrolemath{N} ∪ Σ)*
(i.e., strings of either terminals or non-terminals);

\end{itemize}

\item {} 
\docutilsrolemath{S} is the \emph{start symbol}.

\end{itemize}

A \index{derivation}\textbf{derivation} of a string from a non-terminal \docutilsrolemath{A}
in grammar \docutilsrolemath{G} is the result of successively applying
productions from  \docutilsrolemath{G} to \docutilsrolemath{A}.  For example, \href{\#deriv1}{(50)} is a
derivation of \docutilsroleexample{the dog with a telescope} for the grammar in \href{\#grammar1}{(37)}.
\hypertarget{deriv1}{}\begin{itemize}
\item[(50)] \begin{quote}{\ttfamily \raggedright \noindent
\small
NP~\\
Det~N~PP~\\
the~N~PP~\\
the~dog~PP~\\
the~dog~P~NP~\\
the~dog~with~NP~\\
the~dog~with~Det~N~\\
the~dog~with~a~N~\\
the~dog~with~a~telescope
}\end{quote}
\end{itemize}

Although we have chosen here to expand the leftmost non-terminal
symbol at each stage, this is not obligatory; productions can be
applied in any order. Thus,  derivation \href{\#deriv1}{(50)}
could equally have started off in the following manner:
\hypertarget{deriv2}{}\begin{itemize}
\item[(51)] \begin{quote}{\ttfamily \raggedright \noindent
\small
NP~\\
Det~N~PP~\\
Det~N~P~NP~\\
Det~N~with~NP~\\
...
}\end{quote}
\end{itemize}

We can also write derivation \href{\#deriv1}{(50)} as:
\hypertarget{deriv3}{}\begin{itemize}
\item[(52)] 
\docutilsrolecategory{np} ⇒ \docutilsrolecategory{det n pp}
⇒ \docutilsroleexample{the} \docutilsrolecategory{n pp}
⇒ \docutilsroleexample{the dog} \docutilsrolecategory{pp}
⇒ \docutilsroleexample{the dog} \docutilsrolecategory{p np}
⇒ \docutilsroleexample{the dog with} \docutilsrolecategory{np}
⇒ \docutilsroleexample{the dog with a} \docutilsrolecategory{n}
⇒ \docutilsroleexample{the dog with a telescope}
\end{itemize}

where ⇒ means ``derives in one step''.
We use ⇒* to mean ``derives in zero or more steps'':
\begin{itemize}
\item {} 
α ⇒* α for any string α, and

\item {} 
if α ⇒* β and β
⇒ γ, then α ⇒*
γ.

\end{itemize}

We write \docutilsrolemath{A} ⇒* α to indicate that
α can be derived from  \docutilsrolemath{A}.

In NLTK, context free grammars are defined in the \texttt{parse{\linebreak[0]}.{\linebreak[0]}cfg} module.  The easiest
way to construct a grammar object is from the standard string representation
of grammars.  In \href{\#cfg}{Listing 7.2} we define a grammar and use it to parse a simple
sentence.  You will learn more about parsing in the next section.
\hypertarget{cfg}{}
\begin{pylisting}
\begin{alltt}\small\textbf{grammar = nltk.parse{\_}cfg(\pysrcstring{"{}"{}"}
\pysrcstring{  S -> NP VP}
\pysrcstring{  VP -> V NP | V NP PP}
\pysrcstring{  V -> "saw" | "ate"}
\pysrcstring{  NP -> "John" | "Mary" | "Bob" | Det N | Det N PP}
\pysrcstring{  Det -> "a" | "an" | "the" | "my"}
\pysrcstring{  N -> "dog" | "cat" | "cookie" | "park"}
\pysrcstring{  PP -> P NP}
\pysrcstring{  P -> "in" | "on" | "by" | "with"}
\pysrcstring{  "{}"{}"})}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = \pysrcstring{"Mary saw Bob"}.split()
\pysrcprompt{>{}>{}> }rd{\_}parser = nltk.RecursiveDescentParser(grammar)
\pysrcprompt{>{}>{}> }\pysrckeyword{for} p \pysrckeyword{in} rd{\_}parser.nbest{\_}parse(sent):
\pysrcmore{... }     \pysrckeyword{print} p
\pysrcoutput{(S (NP Mary) (VP (V saw) (NP Bob)))}}\end{alltt}
\caption{Context Free Grammars in NLTK}
\vspace{1ex}
\hrule
\end{pylisting}


%___________________________________________________________________________

\subsection{Exercises}
\label{id72}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ In the recursive descent parser demo, experiment with changing the
sentence to be parsed by selecting \emph{Edit Text} in the \emph{Edit} menu.

\item {} 
☼ Can the grammar in \href{\#grammar1}{(37)} be used to describe sentences that are
more than 20 words in length?

\item {} 
◑ You can modify the grammar in the recursive descent parser demo
by selecting \emph{Edit Grammar}  in the \emph{Edit} menu. Change
the first expansion production, namely \texttt{NP{\linebreak[0]} {\linebreak[0]}-{\linebreak[0]}>{\linebreak[0]} {\linebreak[0]}Det{\linebreak[0]} {\linebreak[0]}N{\linebreak[0]} {\linebreak[0]}PP}, to \texttt{NP{\linebreak[0]} {\linebreak[0]}-{\linebreak[0]}>{\linebreak[0]} {\linebreak[0]}NP
PP}. Using the \emph{Step} button, try to build a parse tree. What happens?

\item {} 
◑ Extend the grammar in \href{\#grammar2}{(40)} with productions that expand prepositions as
intransitive, transitive and requiring a \docutilsrolecategory{pp}
complement. Based on these productions, use the method of the
preceding exercise to draw a tree for the sentence \docutilsroleexample{Lee ran away home}.

\item {} 
◑ Pick some common verbs and complete the following tasks:
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Write a program to find those verbs in the Prepositional Phrase Attachment Corpus
\texttt{nltk{\linebreak[0]}.{\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}ppattach}.  Find any cases where the same verb
exhibits two different attachments, but where the first noun,
or second noun, or preposition, stay unchanged (as we saw in
\href{\#sec-syntactic-ambiguity}{Section (22)}).

\item {} 
Devise CFG grammar productions to cover some of these cases.

\end{list}

\item {} 
★ Write a function that takes a grammar (such as the one defined in
\href{\#cfg}{Listing 7.2}) and returns a random sentence generated by the grammar.
(Use \texttt{grammar{\linebreak[0]}.{\linebreak[0]}start{\linebreak[0]}({\linebreak[0]})} to find the start symbol of the grammar;
\texttt{grammar{\linebreak[0]}.{\linebreak[0]}productions{\linebreak[0]}({\linebreak[0]}lhs{\linebreak[0]})} to get the list of productions from the grammar
that have the specified left-hand side; and \texttt{production{\linebreak[0]}.{\linebreak[0]}rhs{\linebreak[0]}({\linebreak[0]})} to get
the right-hand side of a production.)

\item {} 
★ \textbf{Lexical Acquisition:}
As we saw in \href{\#chap-chunk}{Chapter 6}, it is possible
to collapse chunks down to their chunk label.  When we do this
for sentences involving the word \docutilsroleexample{gave}, we find patterns
such as the following:
\begin{quote}{\ttfamily \raggedright \noindent
\small
gave~NP~\\
gave~up~NP~in~NP~\\
gave~NP~up~\\
gave~NP~NP~\\
gave~NP~to~NP
}\end{quote}
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Use this method to study the complementation patterns of a verb
of interest, and write suitable grammar productions.

\item {} 
Identify some English verbs that are near-synonyms, such as the
\docutilsroleexample{dumped/filled/loaded} example from earlier in this chapter.
Use the chunking method to study the complementation patterns of
these verbs.  Create a grammar to cover these cases.  Can the verbs
be freely substituted for each other, or are their constraints?
Discuss your findings.

\end{list}

\end{list}


%___________________________________________________________________________

\section{Parsing}
\label{parsing}

A \index{parser}\textbf{parser} processes input sentences according to the
productions of a grammar, and builds one or more
constituent structures that conform to the grammar.
A grammar is a declarative specification of well-formedness.
In NLTK, it is just a multi-line string; it is not itself a
program that can be used for anything.
A parser is a procedural interpretation of the grammar.
It searches through the space of trees licensed by a grammar
to find one that has the required sentence along its fringe.

Parsing is important in both linguistics and natural language processing.
A parser permits a grammar to be evaluated against
a potentially large collection of test sentences,
helping linguists to find any problems in their grammatical analysis.
A parser can serve as a model of psycholinguistic processing,
helping to explain the difficulties that humans have with processing
certain syntactic constructions.
Many natural language applications involve parsing at some point;
for example, we would expect the natural language questions
submitted to a question-answering system to undergo parsing as an initial step.

In this section we see two simple parsing algorithms,
a top-down method called recursive descent parsing,
and a bottom-up method called shift-reduce parsing.
\hypertarget{sec-recursive-descent}{}

%___________________________________________________________________________

\subsection{Recursive Descent Parsing}
\label{recursive-descent-parsing}
\label{sec-recursive-descent}

The simplest kind of parser interprets a grammar as a specification
of how to break a high-level goal into several lower-level subgoals.
The top-level goal is to find an \docutilsrolecategory{s}.  The \docutilsrolecategory{s} → \docutilsrolecategory{np vp}
production permits the parser to replace this goal with two subgoals:
find an \docutilsrolecategory{np}, then find a \docutilsrolecategory{vp}.  Each of these subgoals can be
replaced in turn by sub-sub-goals, using productions that have \docutilsrolecategory{np}
and \docutilsrolecategory{vp} on their left-hand side.  Eventually, this expansion
process leads to subgoals such as: find the word \docutilsroleexample{telescope}.  Such
subgoals can be directly compared against the input string, and
succeed if the next word is matched.  If there is no match the parser
must back up and try a different alternative.

The recursive descent parser builds a parse tree during the above
process.  With the initial goal (find an \docutilsrolecategory{s}), the \docutilsrolecategory{s} root node
is created.  As the above process recursively expands its goals using
the productions of the grammar, the parse tree is extended downwards
(hence the name \emph{recursive descent}).  We can see this in action using
the parser demonstration \texttt{nltk{\linebreak[0]}.{\linebreak[0]}draw{\linebreak[0]}.{\linebreak[0]}rdparser{\linebreak[0]}.{\linebreak[0]}demo{\linebreak[0]}({\linebreak[0]})}.
Six stages of the execution of this parser are shown in \href{\#rdparser}{Table 7.3}.
\hypertarget{rdparser}{}
\leavevmode
\begin{longtable}[c]{|p{0.319\locallinewidth}|p{0.308\locallinewidth}|p{0.319\locallinewidth}|}
\hline

\scalebox{0.514286}{\includegraphics{../images/rdparser1.png}}
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Initial stage

\end{list}
 & 
\scalebox{0.514286}{\includegraphics{../images/rdparser2.png}}
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
2nd production

\end{list}
 & 
\scalebox{0.514286}{\includegraphics{../images/rdparser3.png}}
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{2}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Matching \docutilsroleexample{the}

\end{list}
 \\
\hline

\scalebox{0.514286}{\includegraphics{../images/rdparser4.png}}
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{3}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Cannot match \docutilsroleexample{man}

\end{list}
 & 
\scalebox{0.514286}{\includegraphics{../images/rdparser5.png}}
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{4}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Completed parse

\end{list}
 & 
\scalebox{0.514286}{\includegraphics{../images/rdparser6.png}}
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{5}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Backtracking

\end{list}
 \\
\hline
\caption{
Six Stages of a Recursive Descent Parser
}\end{longtable}

During this process, the parser is often forced to choose between several
possible productions.  For example, in going from step 3 to step 4, it
tries to find productions with \docutilsrolecategory{n} on the left-hand side.  The
first of these is \docutilsrolecategory{n} → \docutilsroleexample{man}.  When this does not work
it \index{backtracks}\textit{backtracks}, and tries other \docutilsrolecategory{n} productions in order, under it
gets to \docutilsrolecategory{n} → \docutilsroleexample{dog}, which matches the next word in the
input sentence.  Much later, as shown in step 5, it finds a complete
parse.  This is a tree that covers the entire sentence, without any
dangling edges.  Once a parse has been found, we can get the parser to
look for additional parses.  Again it will backtrack and explore other
choices of production in case any of them result in a parse.

NLTK provides a recursive descent parser:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }rd{\_}parser = nltk.RecursiveDescentParser(grammar)
\pysrcprompt{>{}>{}> }sent = \pysrcstring{'Mary saw a dog'}.split()
\pysrcprompt{>{}>{}> }\pysrckeyword{for} t \pysrckeyword{in} rd{\_}parser.nbest{\_}parse(sent):
\pysrcmore{... }    \pysrckeyword{print} t
\pysrcoutput{(S (NP Mary) (VP (V saw) (NP (Det a) (N dog))))}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

\texttt{RecursiveDescentParser{\linebreak[0]}({\linebreak[0]})} takes an optional parameter \texttt{trace}.
If \texttt{trace} is greater than zero, then the parser will report the steps
that it takes as it parses a text.
\end{boxedminipage}
\end{sffamily}
\end{center}

Recursive descent parsing has three key shortcomings.  First,
left-recursive productions like \docutilsrolecategory{np} → \docutilsrolecategory{np pp} send it
into an infinite loop.  Second, the parser wastes a lot of time
considering words and structures that do not correspond to the input
sentence.  Third, the backtracking process may discard parsed
constituents that will need to be rebuilt again later.  For example,
backtracking over \docutilsrolecategory{vp} → \docutilsrolecategory{v np} will discard the subtree
created for the \docutilsrolecategory{np}.  If the parser then proceeds with \docutilsrolecategory{vp}
→ \docutilsrolecategory{v np pp}, then the \docutilsrolecategory{np} subtree must be created all
over again.

Recursive descent parsing is a kind of \index{top-down parsing}\textbf{top-down parsing}.
Top-down parsers use a grammar to \emph{predict} what the input will be,
before inspecting the input!  However, since the input is available to
the parser all along, it would be more sensible to consider the input
sentence from the very beginning.  This approach is called
\index{bottom-up parsing}\textbf{bottom-up parsing}, and we will see an example in the next section.
\hypertarget{sec-shift-reduce}{}

%___________________________________________________________________________

\subsection{Shift-Reduce Parsing}
\label{shift-reduce-parsing}
\label{sec-shift-reduce}

A simple kind of bottom-up parser is the \index{shift-reduce parser}\textbf{shift-reduce parser}.
In common with all bottom-up parsers, a shift-reduce
parser tries to find sequences of words and phrases that correspond
to the \emph{right hand} side of a grammar production, and replace them
with the left-hand side, until the whole sentence is reduced to
an \docutilsrolecategory{s}.

The shift-reduce parser repeatedly pushes the next input word onto a
stack (\href{\#stacks-and-queues}{Section 5.2.4}); this is the \index{shift}\textbf{shift} operation.
If the top \emph{n} items on the stack match
the \emph{n} items on the right hand side of some production,
then they are all popped off the stack, and the item on the left-hand
side of the production is pushed on the stack.  This replacement of
the top \emph{n} items with a single item is the \index{reduce}\textbf{reduce} operation.
(This reduce operation may only be applied to the top of the stack;
reducing items lower in the stack must be done before later items are
pushed onto the stack.)  The parser finishes when all the input is
consumed and there is only one item remaining on the stack, a parse
tree with an \docutilsrolecategory{s} node as its root.

The shift-reduce parser builds a parse tree during the above process.
If the top of stack holds the word \docutilsroleexample{dog}, and if the grammar has a
production \docutilsrolecategory{n} → \docutilsroleexample{dog}, then the reduce operation causes the word
to be replaced with the parse tree for this production.  For
convenience we will represent this tree as \texttt{N{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]})}.  At a later
stage, if the top of the stack holds two items \texttt{Det{\linebreak[0]}({\linebreak[0]}the{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}N{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]})} and
if the grammar has a production \docutilsrolecategory{np} → \docutilsrolecategory{det n} then the reduce
operation causes these two items to be replaced with \texttt{NP{\linebreak[0]}({\linebreak[0]}Det{\linebreak[0]}({\linebreak[0]}the{\linebreak[0]}){\linebreak[0]},
N{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]}){\linebreak[0]})}.  This process continues until a parse tree for the entire
sentence has been constructed.  We can see this in action using the
parser demonstration \texttt{nltk{\linebreak[0]}.{\linebreak[0]}draw{\linebreak[0]}.{\linebreak[0]}srparser{\linebreak[0]}.{\linebreak[0]}demo{\linebreak[0]}({\linebreak[0]})}.
Six stages of the execution of this parser are shown in \href{\#srparser}{Figure 7.4}.
\hypertarget{srparser}{}
\leavevmode
\begin{longtable}[c]{|p{0.435\locallinewidth}|p{0.435\locallinewidth}|}
\hline

\scalebox{0.514286}{\includegraphics{../images/srparser1.png}}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Initial State

\end{list}
 & 
\scalebox{0.514286}{\includegraphics{../images/srparser2.png}}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
After one shift

\end{list}
 \\
\hline

\scalebox{0.514286}{\includegraphics{../images/srparser3.png}}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{2}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
After reduce shift reduce

\end{list}
 & 
\scalebox{0.514286}{\includegraphics{../images/srparser4.png}}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{3}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
After recognizing the second NP

\end{list}
 \\
\hline

\scalebox{0.514286}{\includegraphics{../images/srparser5.png}}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{4}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Complex NP

\end{list}
 & 
\scalebox{0.514286}{\includegraphics{../images/srparser6.png}}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{5}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Final Step

\end{list}
 \\
\hline
\caption{
Six Stages of a Shift-Reduce Parser
}\end{longtable}

NLTK provides \texttt{ShiftReduceParser{\linebreak[0]}({\linebreak[0]})}, a simple
implementation of a shift-reduce parser.  This parser does not
implement any backtracking, so it is not guaranteed to find a parse
for a text, even if one exists.  Furthermore, it will only find at
most one parse, even if more parses exist.  We can provide an
optional \texttt{trace} parameter that controls how verbosely the
parser reports the steps that it takes as it parses a text:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sr{\_}parse = nltk.ShiftReduceParser(grammar, trace=2)
\pysrcprompt{>{}>{}> }sent = \pysrcstring{'Mary saw a dog'}.split()
\pysrcprompt{>{}>{}> }\pysrckeyword{print} sr{\_}parse.parse(sent)
\pysrcoutput{Parsing 'Mary saw a dog'}
\pysrcoutput{    {[} * Mary saw a dog{]}}
\pysrcoutput{  S {[} 'Mary' * saw a dog{]}}
\pysrcoutput{  R {[} <NP> * saw a dog{]}}
\pysrcoutput{  S {[} <NP> 'saw' * a dog{]}}
\pysrcoutput{  R {[} <NP> <V> * a dog{]}}
\pysrcoutput{  S {[} <NP> <V> 'a' * dog{]}}
\pysrcoutput{  R {[} <NP> <V> <Det> * dog{]}}
\pysrcoutput{  S {[} <NP> <V> <Det> 'dog' * {]}}
\pysrcoutput{  R {[} <NP> <V> <Det> <N> * {]}}
\pysrcoutput{  R {[} <NP> <V> <NP> * {]}}
\pysrcoutput{  R {[} <NP> <VP> * {]}}
\pysrcoutput{  R {[} <S> * {]}}
\pysrcoutput{  (S (NP Mary) (VP (V saw) (NP (Det a) (N dog))))}}\end{alltt}

Shift-reduce parsers have a number of problems.
A shift-reduce parser may fail to parse the sentence, even though the
sentence is well-formed according to the grammar.  In such cases,
there are no remaining input words to shift, and there is no way to
reduce the remaining items on the stack, as exemplified in \href{\#conflict}{Table 7.5}1.
The parser entered this blind alley at an earlier
stage shown in \href{\#conflict}{Table 7.5}2, when it reduced instead of
shifted.  This situation is called a \index{shift-reduce conflict}\textbf{shift-reduce conflict}.  At
another possible stage of processing shown in \href{\#conflict}{Table 7.5}3,
the parser must choose between two possible reductions, both matching
the top items on the stack: \docutilsrolecategory{vp} → \docutilsrolecategory{vp np pp} or \docutilsrolecategory{np} →
\docutilsrolecategory{np pp}.  This situation is called a \index{reduce-reduce conflict}\textbf{reduce-reduce conflict}.
\hypertarget{conflict}{}
\leavevmode
\begin{longtable}[c]{|p{0.505\locallinewidth}|}
\hline

\scalebox{0.514286}{\includegraphics{../images/srparser7.png}}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Dead end

\end{list}
 \\
\hline

\scalebox{0.514286}{\includegraphics{../images/srparser8.png}}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Shift-reduce conflict

\end{list}
 \\
\hline

\scalebox{0.514286}{\includegraphics{../images/srparser9.png}}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{2}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Reduce-reduce conflict

\end{list}
 \\
\hline
\caption{
Conflict in Shift-Reduce Parsing
}\end{longtable}
% To do: diagram showing search tree with success and failure. 

Shift-reduce parsers may implement policies for resolving such
conflicts.  For example, they may address shift-reduce conflicts by
shifting only when no reductions are possible, and they may address
reduce-reduce conflicts by favoring the reduction operation that removes
the most items from the stack.  No such policies are failsafe however.

The advantages of shift-reduce parsers over recursive descent parsers
is that they only build structure that corresponds to the words in the
input.  Furthermore, they only build each sub-structure once,
e.g. \texttt{NP{\linebreak[0]}({\linebreak[0]}Det{\linebreak[0]}({\linebreak[0]}the{\linebreak[0]}){\linebreak[0]},{\linebreak[0]} {\linebreak[0]}N{\linebreak[0]}({\linebreak[0]}man{\linebreak[0]}){\linebreak[0]})} is only built and pushed onto the stack
a single time, regardless of whether it will later be used by the \docutilsrolecategory{vp}
→ \docutilsrolecategory{v np pp} reduction or the \docutilsrolecategory{np} → \docutilsrolecategory{np pp} reduction.


%___________________________________________________________________________

\subsection{The Left-Corner Parser}
\label{the-left-corner-parser}

One of the problems with the recursive descent parser is that it can
get into an infinite loop.  This is because it applies the grammar
productions blindly, without considering the actual input sentence.
A left-corner parser is a hybrid between the bottom-up and top-down
approaches we have seen.

Grammar \href{\#grammar2}{(40)} allows us to produce the following parse of \docutilsroleexample{John saw
Mary}:
\hypertarget{jmtree}{}\begin{itemize}
\item[(53)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-19.pdf}}}\end{itemize}

Recall that the grammar in \href{\#grammar2}{(40)} has the following productions for expanding \docutilsrolecategory{np}:
\begin{itemize}
\item[(54)] \hypertarget{r1}{}\begin{itemize}
\item[a.] 
\docutilsrolecategory{np} → \docutilsrolecategory{dt nom}
\end{itemize}
\hypertarget{r2}{}\begin{itemize}
\item[b.] 
\docutilsrolecategory{np} → \docutilsrolecategory{dt nom pp}
\end{itemize}
\hypertarget{r3}{}\begin{itemize}
\item[c.] 
\docutilsrolecategory{np} → \docutilsrolecategory{propn}
\end{itemize}
\end{itemize}

Suppose we ask you to first look at \href{\#jmtree}{tree (53)}, and then decide
which of the \docutilsrolecategory{np} productions you'd want a recursive descent parser to
apply first — obviously, \href{\#r3}{(54c)} is the right choice! How do you
know that it would be pointless to apply \href{\#r1}{(54a)} or \href{\#r2}{(54b)} instead? Because
neither of these productions will derive a string whose first word is
\docutilsroleexample{John}.  That is, we can easily tell that in a successful
parse of \docutilsroleexample{John saw Mary}, the parser has to expand \docutilsrolecategory{np} in
such a way that \docutilsrolecategory{np} derives the string \docutilsroleexample{John} α. More
generally, we say that a category \docutilsrolemath{B} is a \index{left-corner}\textbf{left-corner} of
a tree rooted in \docutilsrolemath{A} if  \docutilsrolemath{A} ⇒*
\docutilsrolemath{B} α.
\begin{itemize}
\item[(55)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-20.pdf}}}\end{itemize}

A \index{left-corner parser}\textbf{left-corner parser} is a top-down parser with bottom-up filtering.
Unlike an ordinary recursive descent parser, it does not get trapped
in left recursive productions.
Before starting its work, a left-corner parser preprocesses the
context-free grammar to build a table where each row contains two
cells, the first holding a non-terminal, and the second holding the
collection of possible left corners of that non-terminal. \href{\#lc}{Table 7.6}
illustrates this for the grammar from \href{\#grammar2}{(40)}.
\hypertarget{lc}{}
\leavevmode
\begin{longtable}[c]{|p{0.110\locallinewidth}|p{0.342\locallinewidth}|}
\hline
\textbf{
Category
} & \textbf{
Left-Corners (pre-terminals)
} \\
\hline
\endhead

S
 & 
NP
 \\
\hline

NP
 & 
Det, PropN
 \\
\hline

VP
 & 
V
 \\
\hline

PP
 & 
P
 \\
\hline
\caption{
Left-Corners in \href{\#grammar2}{(40)}
}\end{longtable}

Each time a production is considered by the parser, it checks that the
next input word is compatible with at least one of the pre-terminal
categories in the left-corner table.

{[}TODO: \emph{explain how this effects the action of the parser, and why this solves the problem.}{]}


%___________________________________________________________________________

\subsection{Exercises}
\label{id73}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ With pen and paper, manually trace the execution of a recursive descent
parser and a shift-reduce parser, for a CFG you have already seen, or one
of your own devising.

\item {} 
◑ Compare the performance of the top-down, bottom-up, and left-corner
parsers using the same grammar and three grammatical test
sentences. Use \texttt{timeit} to log the amount of time each
parser takes on the same sentence (\href{\#sec-timing}{Section 5.5.4}).  Write a function that runs all
three parsers on all three sentences, and prints a 3-by-3 grid of
times, as well as row and column totals. Discuss your findings.

\item {} 
◑ Read up on ``garden path'' sentences.  How might the computational
work of a parser relate to the difficulty humans have with
processing these sentences?
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}en{\linebreak[0]}.{\linebreak[0]}wikipedia{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}wiki{\linebreak[0]}/{\linebreak[0]}Garden{\_}path{\_}sentence}

\item {} 
★ \textbf{Left-corner parser:} Develop a left-corner parser
based on the recursive descent parser, and inheriting from \texttt{ParseI}.
(Note, this exercise requires knowledge of Python classes, covered
in \href{\#chap-applied-programming}{Chapter 9}.)

\item {} 
★ Extend NLTK's shift-reduce parser to incorporate backtracking, so
that it is guaranteed to find all parses that exist (i.e. it is \index{complete}\textbf{complete}).

\end{list}


%___________________________________________________________________________

\section{Conclusion}
\label{id74}

We began this chapter talking about confusing encounters with grammar
at school.  We just wrote what we wanted to say, and our work was
handed back with red marks showing all our grammar mistakes.  If this
kind of ``grammar'' seems like secret knowledge, the linguistic approach
we have taken in this chapter is quite the opposite: grammatical
structures are made explicit as we build trees on top of sentences.
We can write down the grammar productions, and parsers can build the
trees automatically.  This thoroughly objective approach
is widely referred to as \index{generative grammar}\textbf{generative grammar}.

Note that we have only considered ``toy grammars,'' small grammars that
illustrate the key aspects of parsing.  But there is an obvious
question as to whether the general approach can be scaled up to cover
large corpora of natural languages. How hard would it be to construct
such a set of productions by hand? In general, the answer is: \emph{very
hard}. Even if we allow ourselves to use various formal devices that
give much more succinct representations of grammar productions (some
of which will be discussed in \href{\#chap-advanced-parsing}{Chapter 8}), it is still extremely
difficult to keep control of the complex interactions between the many
productions required to cover the major constructions of a
language. In other words, it is hard to modularize grammars so that
one portion can be developed independently of the other parts. This in
turn means that it is difficult to distribute the task of grammar
writing across a team of linguists. Another difficulty is that as the
grammar expands to cover a wider and wider range of constructions,
there is a corresponding increase in the number of analyses which are
admitted for any one sentence. In other words, ambiguity increases
with coverage.

Despite these problems, there are a number of large collaborative
projects that have achieved interesting and impressive results in
developing rule-based grammars for several languages. Examples are the
Lexical Functional Grammar (LFG) Pargram project
(\href{http://www2.parc.com/istl/groups/nltt/pargram/}{http://www2.parc.com/istl/groups/nltt/pargram/}), the Head-Driven
Phrase Structure Grammar (HPSG) LinGO Matrix framework
(\href{http://www.delph-in.net/matrix/}{http://www.delph-in.net/matrix/}), and the Lexicalized Tree Adjoining
Grammar XTAG Project (\href{http://www.cis.upenn.edu/~xtag/}{http://www.cis.upenn.edu/{\textasciitilde}xtag/}).


%___________________________________________________________________________

\section{Summary (notes)}
\label{summary-notes}
\begin{itemize}
\item {} 
Sentences have internal organization, or constituent structure,
that can be represented using a tree; notable features of constituent
structure are: recursion, heads, complements, modifiers

\item {} 
A grammar is a compact characterization of a potentially infinite set of sentences;
we say that a tree is well-formed according to a grammar, or that a grammar licenses a tree.

\item {} 
Syntactic ambiguity arises when one sentence has more than one syntactic structure
(e.g. prepositional phrase attachment ambiguity).

\item {} 
A parser is a procedure for finding one or more trees corresponding to a grammatically
well-formed sentence.

\item {} 
A simple top-down parser is the recursive descent parser (summary, problems)

\item {} 
A simple bottom-up parser is the shift-reduce parser (summary, problems)

\item {} 
It is difficult to develop a broad-coverage grammar...

\end{itemize}


%___________________________________________________________________________

\section{Further Reading}
\label{id75}

For more examples of parsing with NLTK, please see the
guide at \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}parse{\linebreak[0]}.{\linebreak[0]}html}.

There are many introductory books on syntax. {[}O'Grady1989LI{]}{\_} is a
general introduction to linguistics, while \cite{radford1988tg} provides a
gentle introduction to transformational grammar, and can be
recommended for its coverage of transformational approaches to
unbounded dependency constructions.

\cite{burtonroberts1997as} is very practically oriented textbook on how to
analyze constituency in English, with extensive exemplification and
exercises. \cite{huddleston2002cge} provides an up-to-date and comprehensive analysis of
syntactic phenomena in English.

Chapter 12 of \cite{jurafskymartin2008} covers formal grammars of English;
Sections 13.1-3 cover simple parsing algorithms and techniques
for dealing with ambiguity;
Chapter 16 covers the Chomsky hierarchy and the formal complexity
of natural language.
\begin{itemize}
\item {} 
LALR(1)

\item {} 
Marcus parser

\item {} 
Lexical Functional Grammar (LFG)
\begin{itemize}
\item {} 
\href{http://www2.parc.com/istl/groups/nltt/pargram/}{Pargram project}

\item {} 
\href{http://www.essex.ac.uk/linguistics/lfg/}{LFG Portal}

\end{itemize}

\item {} 
Head-Driven Phrase Structure Grammar (HPSG) \href{http://www.delph-in.net/matrix/}{LinGO Matrix framework}

\item {} 
Lexicalized Tree Adjoining Grammar \href{http://www.cis.upenn.edu/~xtag/}{XTAG Project}

\end{itemize}
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
\hypertarget{chap-advanced-parsing}{}\setcounter{chapter}{7}

%___________________________________________________________________________

\chapter{Chart Parsing and Probabilistic Parsing}
\label{chart-parsing-and-probabilistic-parsing}
\label{chap-advanced-parsing}


%___________________________________________________________________________

\section{Introduction}
\label{id80}

\href{\#chap-parse}{Chapter 7} started with an introduction to constituent
structure in English, showing how words in a sentence group together
in predictable ways. We showed how to describe this structure using
syntactic tree diagrams, and observed that it is sometimes desirable
to assign more than one such tree to a given string. In this case, we
said that the string was structurally ambiguous; and example was \docutilsroleexample{old
men and women}.

Treebanks are language resources in which the syntactic structure of a
corpus of sentences has been annotated, usually by hand. However, we
would also like to be able to produce trees algorithmically. A
context-free phrase structure grammar (CFG) is a formal model for
describing whether a given string can be assigned a particular
constituent structure. Given a set of syntactic categories, the CFG
uses a set of productions to say how a phrase of some category \emph{A} can
be analyzed into a sequence of smaller parts α$_{1}$
... α$_{n}$.  But a grammar is a static description of a set
of strings; it does not tell us what sequence of steps we need to take
to build a constituent structure for a string. For this, we need to
use a parsing algorithm.  We presented two such algorithms: Top-Down Recursive
Descent (\href{\#sec-recursive-descent}{7.5.1}) and Bottom-Up Shift-Reduce
(\href{\#sec-shift-reduce}{7.5.2}). As we pointed out, both parsing approaches suffer
from important shortcomings. The Recursive Descent parser cannot
handle left-recursive productions (e.g., productions such as \docutilsrolecategory{np}
→ \docutilsrolecategory{np pp}), and blindly expands categories top-down without
checking whether they are compatible with the input string. The
Shift-Reduce parser is not guaranteed to find a valid parse for the
input even if one exists, and builds substructure without checking
whether it is globally consistent with the grammar. As we will
describe further below, the Recursive Descent parser is also
inefficient in its search for parses.

So, parsing builds trees over sentences, according to a phrase
structure grammar.  Now, all the examples we gave in \href{\#chap-parse}{Chapter 7} only involved toy grammars containing a handful of
productions. What happens if we try to scale up this approach to deal
with realistic corpora of language? Unfortunately, as the coverage of
the grammar increases and the length of the input sentences grows, the
number of parse trees grows rapidly.  In fact, it grows at an
astronomical rate.

Let's explore this issue with the help of a simple example.
The word
\docutilsroleexample{fish} is both a noun and a verb.  We can make up the sentence
\docutilsroleexample{fish fish fish}, meaning \emph{fish like to fish for other fish}.
(Try this with \docutilsroleexample{police} if you prefer something more sensible.)
Here is a toy grammar for the ``fish'' sentences.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }grammar = nltk.parse{\_}cfg(\pysrcstring{"{}"{}"}
\pysrcmore{... }\pysrcstring{S -> NP V NP}
\pysrcmore{... }\pysrcstring{NP -> NP Sbar}
\pysrcmore{... }\pysrcstring{Sbar -> NP V}
\pysrcmore{... }\pysrcstring{NP -> 'fish'}
\pysrcmore{... }\pysrcstring{V -> 'fish'}
\pysrcmore{... }\pysrcstring{"{}"{}"})}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Remember that our program samples assume you
begin your interactive session or your program with: \texttt{\pysrckeyword{import} {\linebreak[0]}nltk{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}re{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}pprint}
\end{boxedminipage}
\end{sffamily}
\end{center}

Now we can try parsing a longer sentence, \docutilsroleexample{fish fish fish fish
fish}, which amongst other things, means 'fish that other fish
fish are in the habit of fishing fish themselves'. We use the NLTK
chart parser, which is presented later on in this chapter.  This
sentence has two readings.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tokens = {[}\pysrcstring{"fish"}{]} * 5
\pysrcprompt{>{}>{}> }cp = nltk.ChartParser(grammar, nltk.parse.TD{\_}STRATEGY)
\pysrcprompt{>{}>{}> }\pysrckeyword{for} tree \pysrckeyword{in} cp.nbest{\_}parse(tokens):
\pysrcmore{... }    \pysrckeyword{print} tree
\pysrcoutput{(S (NP (NP fish) (Sbar (NP fish) (V fish))) (V fish) (NP fish))}
\pysrcoutput{(S (NP fish) (V fish) (NP (NP fish) (Sbar (NP fish) (V fish))))}}\end{alltt}

As the length of this sentence goes up (3, 5, 7, ...) we get the
following numbers of parse trees:
1; 2; 5; 14; 42; 132; 429; 1,430; 4,862; 16,796; 58,786; 208,012; ...
(These are the \index{Catalan numbers}\textit{Catalan numbers}, which we saw in an exercise
in \href{\#sec-algorithm-design-strategies}{Section 5.5}).
The last of these is for a sentence of length 23, the average length
of sentences in the  WSJ section of Penn Treebank.  For a sentence
of length 50 there would be over 10$^{12}$ parses, and this
is only half the length of the Piglet sentence
(\href{\#sec-more-observations-about-grammar}{Section (17)}),
which young children process effortlessly.
No practical NLP system could construct all millions of trees for a
sentence and choose the appropriate one in the context.
It's clear that humans don't do this either!

Note that the problem is not with our choice of example.
\cite{church1982csa} point out that the syntactic ambiguity of \docutilsrolecategory{pp}
attachment in sentences like \href{\#pp}{(56)} also grows in proportion to the Catalan
numbers.
\hypertarget{pp}{}\begin{itemize}
\item[(56)] 
Put the block in the box on the table.
\end{itemize}

So much for structural ambiguity; what about lexical ambiguity?
As soon as we try to construct a broad-coverage grammar, we
are forced to make lexical entries highly ambiguous for their part of
speech.  In a toy grammar, \docutilsroleexample{a} is only a determiner, \docutilsroleexample{dog} is
only a noun, and \docutilsroleexample{runs} is only a verb.  However, in a
broad-coverage grammar, \titlereference{a} is also a noun (e.g. \docutilsroleexample{part a}),
\docutilsroleexample{dog} is also a verb (meaning to follow closely), and \docutilsroleexample{runs}
is also a noun (e.g. \docutilsroleexample{ski runs}).  In fact, all words can be
referred to by name: e.g. \docutilsroleexample{the verb 'ate' is spelled with three
letters}; in speech we do not need to supply quotation marks.
Furthermore, it is possible to \emph{verb} most nouns.  Thus a parser for a
broad-coverage grammar will be overwhelmed with ambiguity.  Even
complete gibberish will often have a reading, e.g. \docutilsroleexample{the a are of
I}.  As \cite{abney1996sml} has pointed out, this is not word salad but a
grammatical noun phrase, in which \docutilsroleexample{are} is a noun meaning a
hundredth of a hectare (or 100 sq m), and \docutilsroleexample{a} and \docutilsroleexample{I} are
nouns designating coordinates, as shown in \href{\#are}{Figure 8.1}.
\hypertarget{are}{}\begin{figure}[htbp]
\centering

\scalebox{0.102857}{\includegraphics{../images/are.png}}
\caption{The a are of I}\end{figure}

{\noindent}
Even though this phrase is unlikely, it is still grammatical and a
a broad-coverage parser should be able to construct a parse tree
for it.  Similarly, sentences that seem to be
unambiguous, such as \docutilsroleexample{John saw Mary}, turn out to have other
readings we would not have anticipated (as Abney explains).  This
ambiguity is unavoidable, and leads to horrendous inefficiency in
parsing seemingly innocuous sentences.

Let's look more closely at this issue of efficiency.
The top-down recursive-descent parser presented
in \href{\#chap-parse}{Chapter 7} can be very inefficient, since it often builds
and discards the same sub-structure many times over.  We see this
in \href{\#parser-problem}{Figure 8.1}, where a phrase \docutilsroleexample{the block} is identified
as a noun phrase several times, and where this information is discarded
each time we backtrack.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

You should try the recursive-descent parser demo if you haven't
already: \texttt{nltk{\linebreak[0]}.{\linebreak[0]}draw{\linebreak[0]}.{\linebreak[0]}srparser{\linebreak[0]}.{\linebreak[0]}demo{\linebreak[0]}({\linebreak[0]})}
\end{boxedminipage}
\end{sffamily}
\end{center}
\hypertarget{parser-problem}{}
\leavevmode
\begin{longtable}[c]{|p{0.424\locallinewidth}|p{0.424\locallinewidth}|}
\hline
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Initial stage

\end{list}

\scalebox{0.437143}{\includegraphics{../images/findtheblock1.png}}
 & \setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Backtracking

\end{list}

\scalebox{0.437143}{\includegraphics{../images/findtheblock2.png}}
 \\
\hline
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{2}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Failing to match \docutilsroleexample{on}

\end{list}

\scalebox{0.437143}{\includegraphics{../images/findtheblock3.png}}
 & \setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{3}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Completed parse

\end{list}

\scalebox{0.437143}{\includegraphics{../images/findtheblock4.png}}
 \\
\hline
\caption{
Backtracking and Repeated Parsing of Subtrees
}\end{longtable}

In this chapter, we will present two independent methods for dealing with ambiguity.
The first is \docutilsroleemphasis{chart parsing}, which uses the algorithmic technique
of dynamic programming to derive the parses of an ambiguous
sentence more \docutilsroleemphasis{efficiently}.  The second is
\docutilsroleemphasis{probabilistic parsing}, which allows us to \docutilsroleemphasis{rank}
the parses of an ambiguous sentence on the basis of evidence from corpora.
\hypertarget{chart-parsing}{}

%___________________________________________________________________________

\section{Chart Parsing}
\label{id83}
\label{chart-parsing}

\index{chart parsing}

In the introduction to this chapter, we pointed out that the simple
parsers discussed in \href{\#chap-parse}{Chapter 7} suffered from limitations in
both completeness and efficiency. In order to remedy these, we will
apply the algorithm design technique of \index{dynamic programming}\textit{dynamic programming} to
the parsing problem.  As we saw in \href{\#sec-dynamic-programming}{Section 5.5.3},
dynamic programming stores intermediate results and re-uses them when
appropriate, achieving significant efficiency gains. This technique
can be applied to syntactic parsing, allowing us to store
partial solutions to the parsing task and then look them up as
necessary in order to efficiently arrive at a complete solution.
This approach to parsing is known as \index{chart parsing}\textbf{chart parsing}, and is the
focus of this section.


%___________________________________________________________________________

\subsection{Well-Formed Substring Tables}
\label{well-formed-substring-tables}

Let's start off by defining a simple grammar.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }grammar = nltk.parse{\_}cfg(\pysrcstring{"{}"{}"}
\pysrcmore{... }\pysrcstring{S -> NP VP}
\pysrcmore{... }\pysrcstring{PP -> P NP}
\pysrcmore{... }\pysrcstring{NP -> Det N | NP PP}
\pysrcmore{... }\pysrcstring{VP -> V NP | VP PP}
\pysrcmore{... }\pysrcstring{Det -> 'the'}
\pysrcmore{... }\pysrcstring{N -> 'kids' | 'box' | 'floor'}
\pysrcmore{... }\pysrcstring{V -> 'opened'}
\pysrcmore{... }\pysrcstring{P -> 'on'}
\pysrcmore{... }\pysrcstring{"{}"{}"})}\end{alltt}

As you can see, this grammar allows the \docutilsrolecategory{vp}
\docutilsroleexample{opened the box on the floor} to be analyzed in two ways,
depending on where the \docutilsrolecategory{pp} is attached.
\begin{itemize}
\item[(57)] \begin{itemize}
\item[a.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-21.pdf}}}\end{itemize}
\begin{itemize}
\item[b.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-22.pdf}}}\end{itemize}
\end{itemize}

Dynamic programming allows us to build the \docutilsrolecategory{pp} \docutilsroleexample{on the floor}
just once.  The first time we build it we save it in a table, then we look it
up when we need to use it as a subconstituent of either the object \docutilsrolecategory{np} or
the higher \docutilsrolecategory{vp}. This table is known as a
\index{well-formed substring table}\textbf{well-formed substring table} (or WFST for short).
We will show how to construct the WFST bottom-up so as to systematically record
what syntactic constituents have been found.

Let's set our input to be the sentence \docutilsroleexample{the kids opened the box on the
floor}. It is helpful to think of the input as being indexed like a Python
list. We have illustrated this in \href{\#stringpos}{Figure 8.2}.
\hypertarget{stringpos}{}\begin{figure}[htbp]
\centering

\scalebox{0.128571}{\includegraphics{../images/chart_positions.png}}
\caption{Slice Points in the Input String}\end{figure}

{\noindent}
This allows us to say that, for instance, the word \docutilsroleexample{opened} spans
(2, 3) in the input.  This is reminiscent of the slice notation:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tokens = {[}\pysrcstring{"the"}, \pysrcstring{"kids"}, \pysrcstring{"opened"}, \pysrcstring{"the"}, \pysrcstring{"box"}, \pysrcstring{"on"}, \pysrcstring{"the"}, \pysrcstring{"floor"}{]}
\pysrcprompt{>{}>{}> }tokens{[}2:3{]}
\pysrcoutput{{[}'opened'{]}}}\end{alltt}

{\noindent}
In a WFST, we record the position of the words
by filling in cells in a triangular matrix:
the vertical axis will denote the start position of a substring,
while the horizontal axis will denote the end position
(thus \docutilsroleexample{opened} will appear in the cell with coordinates (2, 3)).
To simplify this presentation, we will assume each word has a unique
lexical category, and we will store this (not the word) in the matrix.
So cell (2, 3) will contain the entry \docutilsrolecategory{v}.
More generally, if our input string is
\titlereference{a}$_{1}$\titlereference{a}$_{2}$ ... \titlereference{a}$_{n}$, and our grammar
contains a production of the form \emph{A} → \titlereference{a}$_{i}$, then we add \emph{A} to
the cell (\titlereference{i}-1, \titlereference{i}).

So, for every word in \texttt{tokens}, we can look up in our grammar what
category it belongs to.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }grammar.productions(rhs=tokens{[}2{]})
\pysrcoutput{{[}V -> 'opened'{]}}}\end{alltt}

{\noindent}
For our WFST, we create an \docutilsrolemath{(n-1)} × \docutilsrolemath{(n-1)} matrix
as a list of lists in Python, and initialize it
with the lexical categories of each token, in the \texttt{init{\_}wfst{\linebreak[0]}({\linebreak[0]})}
function in \href{\#wfst}{Listing 8.1}.  We also define a utility function \texttt{display{\linebreak[0]}({\linebreak[0]})}
to pretty-print the WFST for us.
As expected, there is a \docutilsrolecategory{v} in cell (2, 3).
\hypertarget{wfst}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{init{\_}wfst}(tokens, grammar):
    numtokens = len(tokens)
    wfst = {[}{[}\pysrcstring{'.'} \pysrckeyword{for} i \pysrckeyword{in} range(numtokens+1){]} \pysrckeyword{for} j \pysrckeyword{in} range(numtokens+1){]}
    \pysrckeyword{for} i \pysrckeyword{in} range(numtokens):
        productions = grammar.productions(rhs=tokens{[}i{]})
        wfst{[}i{]}{[}i+1{]} = productions{[}0{]}.lhs()
    return wfst
\pysrckeyword{def} \pysrcdefname{complete{\_}wfst}(wfst, tokens, trace=False):
    index = {\{}{\}}
    \pysrckeyword{for} prod \pysrckeyword{in} grammar.productions():
        index{[}prod.rhs(){]} = prod.lhs()
    numtokens = len(tokens)
    \pysrckeyword{for} span \pysrckeyword{in} range(2, numtokens+1):
        \pysrckeyword{for} start \pysrckeyword{in} range(numtokens+1-span):
            end = start + span
            \pysrckeyword{for} mid \pysrckeyword{in} range(start+1, end):
                nt1, nt2 = wfst{[}start{]}{[}mid{]}, wfst{[}mid{]}{[}end{]}
                \pysrckeyword{if} (nt1,nt2) \pysrckeyword{in} index:
                    \pysrckeyword{if} trace:
                        \pysrckeyword{print} \pysrcstring{"{[}{\%}s{]} {\%}3s {[}{\%}s{]} {\%}3s {[}{\%}s{]} ==> {[}{\%}s{]} {\%}3s {[}{\%}s{]}"} {\%} {\textbackslash}
                        (start, nt1, mid, nt2, end, start, index{[}(nt1,nt2){]}, end)
                    wfst{[}start{]}{[}end{]} = index{[}(nt1,nt2){]}
    return wfst
\pysrckeyword{def} \pysrcdefname{display}(wfst, tokens):
    \pysrckeyword{print} \pysrcstring{'{\textbackslash}nWFST '} + \pysrcstring{' '}.join({[}(\pysrcstring{"{\%}-4d"} {\%} i) \pysrckeyword{for} i \pysrckeyword{in} range(1, len(wfst)){]})
    \pysrckeyword{for} i \pysrckeyword{in} range(len(wfst)-1):
        \pysrckeyword{print} \pysrcstring{"{\%}d   "} {\%} i,
        \pysrckeyword{for} j \pysrckeyword{in} range(1, len(wfst)):
            \pysrckeyword{print} \pysrcstring{"{\%}-4s"} {\%} wfst{[}i{]}{[}j{]},
        \pysrckeyword{print}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }wfst0 = init{\_}wfst(tokens, grammar)
\pysrcprompt{>{}>{}> }display(wfst0, tokens)
\pysrcoutput{WFST 1    2    3    4    5    6    7    8}
\pysrcoutput{0    Det  .    .    .    .    .    .    .}
\pysrcoutput{1    .    N    .    .    .    .    .    .}
\pysrcoutput{2    .    .    V    .    .    .    .    .}
\pysrcoutput{3    .    .    .    Det  .    .    .    .}
\pysrcoutput{4    .    .    .    .    N    .    .    .}
\pysrcoutput{5    .    .    .    .    .    P    .    .}
\pysrcoutput{6    .    .    .    .    .    .    Det  .}
\pysrcoutput{7    .    .    .    .    .    .    .    N}
\pysrcoutput{}\pysrcprompt{>{}>{}> }wfst1 = complete{\_}wfst(wfst0, tokens)
\pysrcprompt{>{}>{}> }display(wfst1, tokens)
\pysrcoutput{WFST 1    2    3    4    5    6    7    8}
\pysrcoutput{0    Det  NP   .    .    S    .    .    S}
\pysrcoutput{1    .    N    .    .    .    .    .    .}
\pysrcoutput{2    .    .    V    .    VP   .    .    VP}
\pysrcoutput{3    .    .    .    Det  NP   .    .    NP}
\pysrcoutput{4    .    .    .    .    N    .    .    .}
\pysrcoutput{5    .    .    .    .    .    P    .    PP}
\pysrcoutput{6    .    .    .    .    .    .    Det  NP}
\pysrcoutput{7    .    .    .    .    .    .    .    N}}\end{alltt}
\caption{Acceptor Using Well-Formed Substring Table (based on CYK algorithm)}
\vspace{1ex}
\hrule
\end{pylisting}

Returning to our tabular representation, given that we have \docutilsrolecategory{det}
in cell (0, 1), and \docutilsrolecategory{n} in cell (1, 2), what should we put into
cell (0, 2)? In other words, what syntactic category derives
\docutilsroleexample{the kids}? We have already established that \docutilsrolecategory{Det}
derives \docutilsroleexample{the} and \docutilsrolecategory{n} derives \docutilsroleexample{kids}, so we need to find a
production of the form \emph{A} → \docutilsrolecategory{det} \docutilsrolecategory{n}, that is, a production whose
right hand side matches the categories in the cells we have already
found.  From the grammar, we know that we can enter \docutilsrolecategory{np} in cell (0,2).

More generally, we can enter \emph{A} in \docutilsrolemath{(i, j)} if there
is a production \emph{A} → \emph{B} \emph{C}, and we find
nonterminal \emph{B} in \docutilsrolemath{(i, k)} and \emph{C} in \docutilsrolemath{(k, j)}.
\href{\#wfst}{Listing 8.1} uses this inference step to complete the WFST.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

To help us easily retrieve productions by their right hand
sides, we create an index for the grammar.
This is an example of a space-time trade-off: we do a reverse lookup
on the grammar, instead of having to check through entire list of
productions each time we want to look up via the right hand side.
\end{boxedminipage}
\end{sffamily}
\end{center}

We conclude that there is a parse for the whole input string once
we have constructed an \docutilsrolecategory{s} node that covers the whole input, from
position \titlereference{0} to position \titlereference{8}; i.e., we can conclude that \docutilsrolecategory{s}
⇒* \titlereference{a}$_{1}$\titlereference{a}$_{2}$ ... \titlereference{a}$_{n}$.

Notice that we have not used any built-in parsing functions here.
We've implemented a complete, primitive chart parser from the ground up!


%___________________________________________________________________________

\subsection{Charts}
\label{charts}

By setting \texttt{trace} to \texttt{True} when calling the function \texttt{complete{\_}wfst{\linebreak[0]}({\linebreak[0]})}, we get
additional output.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }wfst1 = complete{\_}wfst(wfst0, tokens, trace=True)
\pysrcoutput{{[}0{]} Det {[}1{]}   N {[}2{]} ==> {[}0{]}  NP {[}2{]}}
\pysrcoutput{{[}3{]} Det {[}4{]}   N {[}5{]} ==> {[}3{]}  NP {[}5{]}}
\pysrcoutput{{[}6{]} Det {[}7{]}   N {[}8{]} ==> {[}6{]}  NP {[}8{]}}
\pysrcoutput{{[}2{]}   V {[}3{]}  NP {[}5{]} ==> {[}2{]}  VP {[}5{]}}
\pysrcoutput{{[}5{]}   P {[}6{]}  NP {[}8{]} ==> {[}5{]}  PP {[}8{]}}
\pysrcoutput{{[}0{]}  NP {[}2{]}  VP {[}5{]} ==> {[}0{]}   S {[}5{]}}
\pysrcoutput{{[}3{]}  NP {[}5{]}  PP {[}8{]} ==> {[}3{]}  NP {[}8{]}}
\pysrcoutput{{[}2{]}   V {[}3{]}  NP {[}8{]} ==> {[}2{]}  VP {[}8{]}}
\pysrcoutput{{[}2{]}  VP {[}5{]}  PP {[}8{]} ==> {[}2{]}  VP {[}8{]}}
\pysrcoutput{{[}0{]}  NP {[}2{]}  VP {[}8{]} ==> {[}0{]}   S {[}8{]}}}\end{alltt}

{\noindent} For example, this says that since we found \texttt{Det} at
\texttt{wfst{\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}1{\linebreak[0]}{]}} and \texttt{N} at \texttt{wfst{\linebreak[0]}{[}{\linebreak[0]}1{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}2{\linebreak[0]}{]}}, we can add \texttt{NP} to
\texttt{wfst{\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}2{\linebreak[0]}{]}}.
The same information can be represented in a directed acyclic graph,
as shown in \href{\#chartinit}{Figure 8.2}(a). This graph is usually called a \index{chart}\textbf{chart}.
\href{\#chartinit}{Figure 8.2}(b) is the corresponding graph representation, where
we add a new edge labeled \docutilsrolecategory{np} to cover the input from \titlereference{0} to \titlereference{2}.
\hypertarget{chartinit}{}
\leavevmode
\begin{longtable}[c]{|p{0.795\locallinewidth}|}
\hline
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Initialized WFST

\end{list}

\scalebox{0.128571}{\includegraphics{../images/chart_init0.png}}
 \\
\hline
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Adding an \docutilsrolecategory{np} Edge

\end{list}

\scalebox{0.128571}{\includegraphics{../images/chart_init1.png}}
 \\
\hline
\caption{
A Graph Representation for the WFST
}\end{longtable}

(Charts are more general than the WFSTs we have seen, since they
can hold multiple hypotheses for a given span.)

A WFST is a data structure that can be used by a variety of parsing
algorithms. The particular method for constructing a WFST that we
have just seen and has some shortcomings.
First, as you can see, the WFST is not itself a parse tree, so the technique is
strictly speaking \index{recognizing}\textbf{recognizing} that a sentence is admitted by a
grammar, rather than parsing it.
Second, it requires every non-lexical grammar production to be
\docutilsroleemphasis{binary} (see \href{\#sec-normal-forms}{Section 8.5.1}).
Although it is possible to convert an arbitrary CFG into this form,
we would prefer to use an approach without such a requirement.
Third, as a bottom-up approach it is potentially wasteful, being
able to propose constituents in locations that would not be licensed by
the grammar.
Finally, the WFST did not represent the structural ambiguity in
the sentence (i.e. the two verb phrase readings).  The \docutilsrolecategory{vp}
in cell (\titlereference{2,8}) was actually entered twice, once for a \docutilsrolecategory{v np}
reading, and once for a \docutilsrolecategory{vp pp} reading.
In the next section we will address these issues.


%___________________________________________________________________________

\subsection{Exercises}
\label{id84}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Consider the sequence of words:
\docutilsroleexample{Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo}.
This is a grammatically correct sentence, as explained at
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}en{\linebreak[0]}.{\linebreak[0]}wikipedia{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}wiki{\linebreak[0]}/{\linebreak[0]}Buffalo{\_}buffalo{\_}Buffalo{\_}buffalo{\_}buffalo{\_}buffalo{\_}Buffalo{\_}buffalo{\linebreak[0]}.}
Consider the tree diagram presented on this Wikipedia page, and write down a suitable
grammar.  Normalize case to lowercase, to simulate the problem that a listener has when hearing
this sentence.  Can you find other parses for this sentence?
How does the number of parse trees grow as the sentence gets longer?
(More examples of these sentences can be found at \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}en{\linebreak[0]}.{\linebreak[0]}wikipedia{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}wiki{\linebreak[0]}/{\linebreak[0]}List{\_}of{\_}homophonous{\_}phrases}).

\item {} 
◑ Consider the algorithm in \href{\#wfst}{Listing 8.1}.  Can you explain why
parsing context-free grammar is proportional to \titlereference{n}$^{3}$?

\item {} 
◑ Modify the functions \texttt{init{\_}wfst{\linebreak[0]}({\linebreak[0]})} and \texttt{complete{\_}wfst{\linebreak[0]}({\linebreak[0]})} so
that the contents of each cell in the WFST is a set of
non-terminal symbols rather than a single non-terminal.

\item {} 
★ Modify the functions \texttt{init{\_}wfst{\linebreak[0]}({\linebreak[0]})} and \texttt{complete{\_}wfst{\linebreak[0]}({\linebreak[0]})} so
that when a non-terminal symbol is added to a cell in the WFST, it includes
a record of the cells from which it was derived. Implement a
function that will convert a WFST in this form to a parse tree.

\end{list}


%___________________________________________________________________________

\section{Active Charts}
\label{active-charts}

One important aspect of the tabular approach to parsing can be seen more
clearly if we look at the graph representation: given our grammar, there are two
different ways to derive a top-level \docutilsrolecategory{vp} for the input, as shown in
\href{\#chartnp}{Table 8.3}(a,b).
In our graph representation, we simply combine the
two sets of edges to yield \href{\#chartnp}{Table 8.3}(c).
\hypertarget{chartnp}{}
\leavevmode
\begin{longtable}[c]{|p{0.679\locallinewidth}|}
\hline
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
\docutilsrolecategory{vp} → \docutilsrolecategory{v} \docutilsrolecategory{np}

\end{list}

\scalebox{0.128571}{\includegraphics{../images/chart_np0.png}}
 \\
\hline
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
\docutilsrolecategory{vp} → \docutilsrolecategory{vp} \docutilsrolecategory{pp}

\end{list}

\scalebox{0.128571}{\includegraphics{../images/chart_np1.png}}
 \\
\hline
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{2}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Merged Chart

\end{list}

\scalebox{0.128571}{\includegraphics{../images/chart_np2.png}}
 \\
\hline
\caption{
Combining Multiple Parses in a Single Chart
}\end{longtable}

{\noindent} However, given a WFST we cannot necessarily read off
the justification for adding a particular edge.
For example, in \href{\#chartnp}{8.3}(b), \texttt{{[}{\linebreak[0]}Edge{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}VP{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}2{\linebreak[0]}:{\linebreak[0]}8{\linebreak[0]}{]}}
might owe its existence to a production \docutilsrolecategory{vp} → \docutilsrolecategory{v np pp}.
Unlike phrase structure trees, a WFST does not encode a relation of
immediate dominance. In order to make such information available, we
can label edges not just with a non-terminal category, but with the
whole production that justified the addition of the edge. This is
illustrated in \href{\#chartprods}{Figure 8.3}.
\hypertarget{chartprods}{}\begin{figure}[htbp]
\centering

\scalebox{0.128571}{\includegraphics{../images/chart_prods.png}}
\caption{Chart Annotated with Productions}\end{figure}

In general, a chart parser hypothesizes constituents (i.e. adds edges)
based on the grammar, the tokens, and the constituents already found.
Any constituent that is compatible with the current knowledge can be hypothesized;
even though many of these hypothetical constituents will never be used in
the final result.  A WFST just records these hypotheses.

All of the edges that we've seen so far represent complete
constituents.  However, as we will see, it is helpful to hypothesize
\emph{incomplete} constituents.  For example, the work done by a
parser in processing the production \emph{VP} → \emph{V} \emph{NP} \emph{PP}
can be reused when processing \emph{VP} → \emph{V} \emph{NP}.
Thus, we will record the
hypothesis that ``the \docutilsrolecategory{v} constituent \docutilsroleexample{likes} is the beginning of a \docutilsrolecategory{vp}.''

We can record such hypotheses by adding a \index{dot}\textbf{dot} to the
edge's right hand side.  Material to the left of the dot specifies
what the constituent starts with; and material to the
right of the dot specifies what still needs to be found in order
to complete the constituent.  For example, the edge in the
\href{\#dottededge}{Figure 8.4} records the hypothesis that ``a \docutilsrolecategory{vp} starts with the \docutilsrolecategory{v}
\docutilsroleexample{likes}, but still needs an \docutilsrolecategory{np} to become complete'':
\hypertarget{dottededge}{}\begin{figure}[htbp]
\centering

\scalebox{0.154286}{\includegraphics{../images/chart_intro_dottededge.png}}
\caption{Chart Containing Incomplete VP Edge}\end{figure}

{\noindent} These \index{dotted edges}\textbf{dotted edges} are used to record all of the hypotheses that a
chart parser makes about constituents in a sentence.  Formally
a dotted edge {[}\titlereference{A} → \titlereference{c}$_{1}$ … \titlereference{c}$_{d}$ • \titlereference{c}$_{d+1}$ … \titlereference{c}$_{n}$, (\emph{i}, \emph{j}){]}
records the hypothesis that a constituent of type \titlereference{A} with span (\titlereference{i, j})
starts with children \titlereference{c}$_{1}$ … \titlereference{c}$_{d}$, but still
needs children \titlereference{c}$_{d+1}$ … \titlereference{c}$_{n}$ to be complete
(\titlereference{c}$_{1}$ … \titlereference{c}$_{d}$ and \titlereference{c}$_{d+1}$ … \titlereference{c}$_{n}$ may be empty).
If \docutilsrolemath{d = n}, then \titlereference{c}$_{d+1}$ … \titlereference{c}$_{n}$ is
empty and the edge represents a complete constituent and is called
a \index{complete edge}\textbf{complete edge}.  Otherwise, the edge represents an incomplete
constituent, and is called an \index{incomplete edge}\textbf{incomplete edge}.  In \href{\#chart-terms}{Figure 8.4}(a), {[}\docutilsrolecategory{vp} → \docutilsrolecategory{v} \docutilsrolecategory{np} •, (1, 3){]} is a
complete edge, and {[}\docutilsrolecategory{vp} → \docutilsrolecategory{v} • \docutilsrolecategory{np}, (1, 2){]} is
an incomplete edge.

If \docutilsrolemath{d = 0}, then \titlereference{c}$_{1}$ … \titlereference{c}$_{n}$ is empty
and the edge is called a \index{self-loop edge}\textbf{self-loop edge}.
This is illustrated in \href{\#chart-terms}{Table 8.4}(b).
If a complete edge spans the entire sentence, and has the grammar's
start symbol as its left-hand side, then the edge is called a \index{parse
edge}\textbf{parse
edge}, and it encodes one or more parse trees for the sentence.  In
\href{\#chart-terms}{Table 8.4}(c), {[}\docutilsrolecategory{s} → \docutilsrolecategory{np} \docutilsrolecategory{vp} •, (0, 3){]} is a parse edge.
\hypertarget{chart-terms}{}
\leavevmode
\begin{longtable}[c]{|p{0.319\locallinewidth}|p{0.307\locallinewidth}|p{0.307\locallinewidth}|}
\hline

a. Incomplete Edge
\scalebox{0.128571}{\includegraphics{../images/chart_intro_incomplete.png}}
 & 
b. Self Loop Edge
\scalebox{0.128571}{\includegraphics{../images/chart_intro_selfloop.png}}
 & 
c. Parse Edge
\scalebox{0.128571}{\includegraphics{../images/chart_intro_parseedge.png}}
 \\
\hline
\caption{
Chart Terminology
}\end{longtable}


%___________________________________________________________________________

\subsection{The Chart Parser}
\label{the-chart-parser}

To parse a sentence, a chart parser first creates an empty chart
spanning the sentence.  It then finds edges that are licensed by its
knowledge about the sentence, and adds them to the chart one at a time
until one or more parse edges are found.  The edges that it adds can
be licensed in one of three ways:
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
The \emph{input} can license an edge.  In particular, each word \titlereference{w}$_{i}$
in the input licenses the complete edge {[}\titlereference{w}$_{i}$ →
•, (\emph{i}, \emph{i}+1){]}.

\item {} 
The \emph{grammar} can license an edge.  In particular, each grammar
production A → α licenses the self-loop edge {[}\emph{A} →
• α, (\emph{i}, \emph{i}){]} for every \emph{i}, 0 ≤ \emph{i} {\textless} \emph{n}.

\item {} 
The \emph{current chart contents} can license an edge.

\end{list}

However, it is not wise to add \docutilsroleemphasis{all} licensed edges to the chart,
since many of them will not be used in any complete parse.  For
example, even though the edge in the following chart is licensed (by
the grammar), it will never be used in a complete parse:
\hypertarget{uselessedge}{}\begin{figure}[htbp]
\centering

\scalebox{0.154286}{\includegraphics{../images/chart_useless_edge.png}}
\caption{Chart Containing Redundant Edge}\end{figure}

Chart parsers therefore use a set of \index{rules}\textbf{rules} to heuristically decide
when an edge should be added to a chart.  This set of rules, along
with a specification of when they should be applied, forms a
\index{strategy}\textbf{strategy}.


%___________________________________________________________________________

\subsection{The Fundamental Rule}
\label{the-fundamental-rule}

One rule is particularly important, since it is used by every chart
parser: the \index{Fundamental Rule}\textbf{Fundamental Rule}.
This rule is used to combine an
incomplete edge that's expecting a nonterminal \emph{B} with a following, complete
edge whose left hand side is \emph{B}.
\hypertarget{fundamental-rule}{}\begin{itemize}
\item[(58)] 
\index{Fundamental Rule}\textbf{Fundamental Rule}
\begin{quote}{\ttfamily \raggedright \noindent
\small
If~the~chart~contains~the~edges~\\
~~{[}\emph{A}~→~α~•~\emph{B}~β~,~(\emph{i},~\emph{j}){]}~\\
~~{[}\emph{B}~→~γ~•~,~(\emph{j},~\emph{k}){]}~\\
then~add~the~new~edge~\\
~~{[}\emph{A}~→~α~\emph{B}~•~β~,~(\emph{i},~\emph{k}){]}~\\
where~α,~β,~and~γ~are~(possibly~empty)~sequences~\\
of~terminals~or~non-terminals
}\end{quote}
\end{itemize}

Note that the dot has moved one place to the right, and the span of this new edge is the combined
span of the other two.  Note also that in adding this new edge we do not remove the other two,
because they might be used again.

A somewhat more intuitive version of the operation of the Fundamental
Rule can be given using chart diagrams. Thus, if we have a chart of
the form shown in \href{\#fr}{Table 8.5}(a),
then we can add a new complete edge as shown in \href{\#fr}{Table 8.5}(b).
\hypertarget{fr}{}
\leavevmode
\begin{longtable}[c]{|p{0.377\locallinewidth}|p{0.377\locallinewidth}|}
\hline

a. Input
\scalebox{0.154286}{\includegraphics{../images/chart_fr1.png}}
 & 
b. Output
\scalebox{0.154286}{\includegraphics{../images/chart_fr2.png}}
 \\
\hline
\caption{
Fundamental Rule
}\end{longtable}
\footnotetext[2]{
The Fundamental Rule corresponds to the Completer
function in the Earley algorithm; cf. \cite{jurafskymartin2008}.
}


%___________________________________________________________________________

\subsection{Bottom-Up Parsing}
\label{bottom-up-parsing}

As we saw in \href{\#chap-parse}{Chapter 7}, bottom-up parsing starts from the
input string, and tries to find sequences of words and phrases that
correspond to the \emph{right hand} side of a grammar production. The
parser then replaces these with the left-hand side of the production,
until the whole sentence is reduced to an \docutilsrolecategory{S}.  Bottom-up chart
parsing is an extension of this approach in which hypotheses about
structure are recorded as edges on a chart. In terms of our earlier
terminology, bottom-up chart parsing can be seen as a parsing
strategy; in other words, bottom-up is a particular choice of
heuristics for adding new edges to a chart.

The general procedure for chart parsing is
inductive: we start with a base case, and then show how we can move
from a given state of the chart to a new state. Since we are working
bottom-up, the base case for our induction will be determined by the
words in the input string, so we add new edges for each word.  Now,
for the induction step, suppose the chart contains an edge labeled
with constituent \emph{A}. Since we are working bottom-up, we want to build
constituents that can have an \emph{A} as a daughter. In other words, we
are going to look for productions of the form \emph{B} → \emph{A} β and use
these to label new edges.

Let's look at the procedure a bit more formally.  To create a
bottom-up chart parser, we add to the Fundamental Rule two new rules:
the \index{Bottom-Up Initialization Rule}\textbf{Bottom-Up Initialization Rule}; and the \index{Bottom-Up Predict
Rule}\textbf{Bottom-Up Predict
Rule}.
The Bottom-Up Initialization Rule says to add all edges licensed by
the input.
\hypertarget{bottom-up-initialization-rule}{}\begin{itemize}
\item[(59)] 
\index{Bottom-Up Initialization Rule}\textbf{Bottom-Up Initialization Rule}
\begin{quote}{\ttfamily \raggedright \noindent
\small
For~every~word~w$_{i}$~add~the~edge~\\
~~{[}\titlereference{w}$_{i}$~→~~•~,~(\emph{i},~\emph{i}+1){]}
}\end{quote}
\end{itemize}

\href{\#bu-init-chart}{Table 8.6}(a) illustrates this rule using the chart notation,
while \href{\#bu-init-chart}{Table 8.6}(b) shows the bottom-up
initialization for the input \docutilsroleexample{Lee likes coffee}.
\hypertarget{bu-init-chart}{}
\leavevmode
\begin{longtable}[c]{|p{0.365\locallinewidth}|p{0.342\locallinewidth}|}
\hline

a. Generic
\scalebox{0.154286}{\includegraphics{../images/chart_bu_init.png}}
 & 
b. Example
\scalebox{0.154286}{\includegraphics{../images/chart_bu_ex1.png}}
 \\
\hline
\caption{
Bottom-Up Initialization Rule
}\end{longtable}

Notice that the dot on the right hand side of these productions
is telling us that we have complete edges for the lexical items. By
including this information, we
can give a uniform statement of how the Fundamental Rule operates in
Bottom-Up parsing, as we will shortly see.

Next, suppose the chart contains a complete edge \emph{e} whose
left hand category is \emph{A}. Then the Bottom-Up Predict Rule requires the
parser to add a self-loop edge at the left boundary of \emph{e}
for each grammar production whose right hand side begins with category
\emph{A}.
\hypertarget{bottom-up-predict-rule}{}\begin{itemize}
\item[(60)] 
\index{Bottom-Up Predict Rule}\textbf{Bottom-Up Predict Rule}
\begin{quote}{\ttfamily \raggedright \noindent
\small
If~the~chart~contains~the~complete~edge~\\
~~{[}\emph{A}~→~α~•~,~(\emph{i},~\emph{j}){]}~\\
and~the~grammar~contains~the~production~\\
~~\emph{B}~→~\emph{A}~β~\\
then~add~the~self-loop~edge~\\
~~{[}\emph{B}~→~~•~\emph{A}~β~,~(\emph{i},~\emph{i}){]}
}\end{quote}
\end{itemize}

Graphically, if the chart looks as in \href{\#bu-predict-chart}{Figure 8.7}(a),
then the Bottom-Up Predict Rule tells the parser to augment the chart
as shown in \href{\#bu-predict-chart}{Figure 8.7}(b).
\hypertarget{bu-predict-chart}{}
\leavevmode
\begin{longtable}[c]{|p{0.342\locallinewidth}|p{0.319\locallinewidth}|}
\hline

a. Input
\scalebox{0.154286}{\includegraphics{../images/chart_bu_predict1.png}}
 & 
b. Output
\scalebox{0.154286}{\includegraphics{../images/chart_bu_predict2.png}}
 \\
\hline
\caption{
Bottom-Up Prediction Rule
}\end{longtable}

To continue our earlier example, let's suppose that our grammar
contains the lexical productions shown in \href{\#lexical-productions}{(61a)}.
This allows us to add three self-loop edges to the chart, as
shown in \href{\#buex2}{(61b)}.
\begin{itemize}
\item[(61)] \hypertarget{lexical-productions}{}\begin{itemize}
\item[a.] 
\docutilsrolecategory{np} → \docutilsroleexample{Lee} {\textbar} \docutilsroleexample{coffee}

\docutilsrolecategory{v} → \docutilsroleexample{likes}
\end{itemize}
\hypertarget{buex2}{}\begin{itemize}
\item[b.] \end{itemize}
\scalebox{0.154286}{\includegraphics{../images/chart_bu_ex2.png}}\end{itemize}

Once our chart contains an instance of the pattern shown in \href{\#bu-predict-chart}{Figure 8.7}(b),
we can use the Fundamental Rule to add an edge where we have ``moved the dot'' one
position to the right, as shown in \href{\#bu-fr}{Figure 8.8}
(we have omitted the self-loop edges for simplicity.)
\hypertarget{bu-fr}{}
\leavevmode
\begin{longtable}[c]{|p{0.272\locallinewidth}|p{0.365\locallinewidth}|}
\hline
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Generic

\end{list}

\scalebox{0.154286}{\includegraphics{../images/chart_bu_fr.png}}
 & \setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Example

\end{list}

\scalebox{0.154286}{\includegraphics{../images/chart_bu_ex3.png}}
 \\
\hline
\caption{
Fundamental Rule used in Bottom-Up Parsing
}\end{longtable}

{\noindent} We will now be able to add new self-loop edges such as
{[}\docutilsrolecategory{s} →  • \docutilsrolecategory{np} \docutilsrolecategory{vp}, (0, 0){]} and
{[}\docutilsrolecategory{vp} →  • \docutilsrolecategory{vp} \docutilsrolecategory{np}, (1, 1){]}, and use these to
build more complete edges.

Using these three productions, we can parse a sentence as shown in
\href{\#bottom-up-strategy}{(62)}.
\hypertarget{bottom-up-strategy}{}\begin{itemize}
\item[(62)] 
\index{Bottom-Up Strategy}\textbf{Bottom-Up Strategy}
\begin{quote}{\ttfamily \raggedright \noindent
\small
Create~an~empty~chart~spanning~the~sentence.~\\
Apply~the~Bottom-Up~Initialization~Rule~to~each~word.~\\
Until~no~more~edges~are~added:~\\
~~Apply~the~Bottom-Up~Predict~Rule~everywhere~it~applies.~\\
~~Apply~the~Fundamental~Rule~everywhere~it~applies.~\\
Return~all~of~the~parse~trees~corresponding~to~the~parse~edges~in~the~chart.
}\end{quote}
\end{itemize}

NLTK provides a useful interactive tool for visualizing the way in which charts
are built, \texttt{nltk{\linebreak[0]}.{\linebreak[0]}draw{\linebreak[0]}.{\linebreak[0]}chart{\linebreak[0]}.{\linebreak[0]}demo{\linebreak[0]}({\linebreak[0]})}.
The tool comes with a pre-defined input string and grammar, but both
of these can be readily modified with options inside the \emph{Edit} menu.
\href{\#budemo1}{Figure 8.6} illustrates a window after the grammar has been updated:
\hypertarget{budemo1}{}\begin{figure}[htbp]
\centering

\scalebox{0.514286}{\includegraphics{../images/chart_demo1.png}}
\caption{Modifying the \titlereference{demo()} grammar}\end{figure}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

To get the symbol ⇒ illustrated in \href{\#budemo1}{Figure 8.6}. you just have to type the keyboard characters \titlereference{'-{\textgreater}'}.
\end{boxedminipage}
\end{sffamily}
\end{center}

\href{\#budemo2}{Figure 8.7} illustrates the tool interface. In order to invoke a
rule, you simply click one of the green buttons at the bottom of the window.
We show the state of the chart on the input \docutilsroleexample{Lee
likes coffee} after three applications of the Bottom-Up
Initialization Rule, followed by successive applications of the Bottom-Up Predict
Rule and the Fundamental Rule.
\hypertarget{budemo2}{}\begin{figure}[htbp]
\centering

\scalebox{0.514286}{\includegraphics{../images/chart_demo2.png}}
\caption{Incomplete chart for \docutilsroleexample{Lee likes coffee}}\end{figure}

Notice that in the topmost pane of the window, there is a partial tree
showing that we have constructed an \docutilsrolecategory{s} with an \docutilsrolecategory{np} subject
in the expectation that we will be able to find a \docutilsrolecategory{vp}.


%___________________________________________________________________________

\subsection{Top-Down Parsing}
\label{top-down-parsing}

Top-down chart parsing works in a similar way to the recursive descent
parser discussed in \href{\#chap-parse}{Chapter 7}, in that it starts off with
the top-level goal of finding an \docutilsrolecategory{s}. This goal is then broken into
the subgoals of trying to find constituents such as \docutilsrolecategory{np} and
\docutilsrolecategory{vp} that can be immediately dominated by  \docutilsrolecategory{s}.
To create a top-down chart parser, we use the Fundamental Rule as before plus
three other rules: the \index{Top-Down Initialization Rule}\textbf{Top-Down Initialization Rule}, the \index{Top-Down
Expand Rule}\textbf{Top-Down
Expand Rule}, and the \index{Top-Down Match Rule}\textbf{Top-Down Match Rule}.
The Top-Down Initialization Rule in \href{\#td-init-rule}{(63)}
captures the fact that the root of any
parse must be the start symbol \docutilsrolecategory{s}.
It is illustrated graphically in \href{\#td-init-chart}{Table 8.9}.
\hypertarget{td-init-rule}{}\begin{itemize}
\item[(63)] 
\index{Top-Down Initialization Rule}\textbf{Top-Down Initialization Rule}
\begin{quote}{\ttfamily \raggedright \noindent
\small
For~every~grammar~production~of~the~form:~\\
~~\docutilsrolecategory{s}~→~α~\\
add~the~self-loop~edge:~\\
~~{[}\docutilsrolecategory{s}~→~~•~α,~(0,~0){]}
}\end{quote}
\end{itemize}
\hypertarget{td-init-chart}{}
\leavevmode
\begin{longtable}[c]{|p{0.272\locallinewidth}|p{0.365\locallinewidth}|}
\hline
\setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Generic

\end{list}

\scalebox{0.154286}{\includegraphics{../images/chart_td_init.png}}
 & \setcounter{listcnt0}{0}
\begin{list}{\alph{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Example

\end{list}

\scalebox{0.154286}{\includegraphics{../images/chart_td_ex1.png}}
 \\
\hline
\caption{
Top-Down Initialization Rule
}\end{longtable}

As we mentioned before, the dot on the right hand side of a production
records how far our goals have been satisfied. So in \href{\#td-init-chart}{Figure 8.9}(b), we are predicting that we will be able to find an \docutilsrolecategory{np} and a
\docutilsrolecategory{vp}, but have not yet satisfied these subgoals. So how do we
pursue them? In order to find an  \docutilsrolecategory{np}, for instance, we need to
invoke a production that has \docutilsrolecategory{np} on its left hand side. The step
of adding the required edge to the chart is accomplished with the
Top-Down Expand Rule \href{\#td-expand-rule}{(64)}.
This tells us that if our chart contains an incomplete
edge whose dot is followed by a nonterminal \emph{B}, then the parser
should add any self-loop edges licensed by the grammar whose left-hand
side is \emph{B}.
\hypertarget{td-expand-rule}{}\begin{itemize}
\item[(64)] 
\index{Top-Down Expand Rule}\textbf{Top-Down Expand Rule}
\begin{quote}{\ttfamily \raggedright \noindent
\small
If~the~chart~contains~the~incomplete~edge~\\
~~{[}\emph{A}~→~α~•~\emph{B}~β~,~(\emph{i},~\emph{j}){]}~\\
then~for~each~grammar~production~\\
~~\emph{B}~→~γ~\\
add~the~edge~\\
~~{[}\emph{B}~→~•~γ~,~(\emph{j},~\emph{j}){]}
}\end{quote}
\end{itemize}

{\noindent} Thus, given a chart that looks like the one in \href{\#td-expand-chart}{Table 8.10}(a),
the Top-Down Expand Rule augments it with the edge shown in \href{\#td-expand-chart}{Table 8.10}(b).
In terms of our running example, we now have the chart shown in \href{\#td-expand-chart}{Table 8.10}(c).
\hypertarget{td-expand-chart}{}
\leavevmode
\begin{longtable}[c]{|p{0.266\locallinewidth}|p{0.289\locallinewidth}|p{0.391\locallinewidth}|}
\hline

a. Input
\scalebox{0.154286}{\includegraphics{../images/chart_td_expand1.png}}
 & 
b. Output
\scalebox{0.154286}{\includegraphics{../images/chart_td_expand2.png}}
 & 
c. Example
\scalebox{0.154286}{\includegraphics{../images/chart_td_ex2.png}}
 \\
\hline
\caption{
Top-Down Expand Rule
}\end{longtable}

The Top-Down Match rule allows the predictions of the grammar to be
matched against the input string. Thus, if the chart contains an incomplete
edge whose dot is followed by a terminal \emph{w}, then the parser should
add an edge if the terminal corresponds to the current input symbol.
\hypertarget{top-down-match-rule}{}\begin{itemize}
\item[(65)] 
\index{Top-Down Match Rule}\textbf{Top-Down Match Rule}
\begin{quote}{\ttfamily \raggedright \noindent
\small
If~the~chart~contains~the~incomplete~edge~\\
~~{[}\emph{A}~→~α~•~w$_{j}$~β,~(\emph{i},~\emph{j}){]},~\\
where~w$_{j}$~is~the~\emph{j}~$^{th}$~word~of~the~input,~\\
then~add~a~new~complete~edge~\\
~~{[}\titlereference{w}$_{j}$~→~•~,~(\emph{j},~\emph{j}+1){]}
}\end{quote}
\end{itemize}

{\noindent} Graphically, the Top-Down Match rule takes us from
\href{\#td-match-chart}{Table 8.11}(a), to \href{\#td-match-chart}{Table 8.11}(b).
\hypertarget{td-match-chart}{}
\leavevmode
\begin{longtable}[c]{|p{0.319\locallinewidth}|p{0.296\locallinewidth}|}
\hline

a. Input
\scalebox{0.154286}{\includegraphics{../images/chart_td_match1.png}}
 & 
b. Output
\scalebox{0.154286}{\includegraphics{../images/chart_td_match2.png}}
 \\
\hline
\caption{
Top-Down Match Rule
}\end{longtable}

\href{\#td-example}{Figure 8.12}(a) illustrates how our example chart after applying the
Top-Down Match rule.
What rule is relevant now? The Fundamental Rule. If we remove the
self-loop edges from \href{\#td-example}{Figure 8.12}(a) for simplicity, the
Fundamental Rule gives us \href{\#td-example}{Figure 8.12}(b).
\hypertarget{td-example}{}
\leavevmode
\begin{longtable}[c]{|p{0.458\locallinewidth}|p{0.435\locallinewidth}|}
\hline

a. Apply Top-Down Match Rule
\scalebox{0.154286}{\includegraphics{../images/chart_td_ex3.png}}
 & 
b. Apply Fundamental Rule
\scalebox{0.154286}{\includegraphics{../images/chart_td_ex4.png}}
 \\
\hline
\caption{
Top-Down Example (cont)
}\end{longtable}

Using these four rules, we can parse a sentence top-down as shown in
\href{\#top-down-strategy}{(66)}.
% I don't see why 'each word': Apply the Top-Down Initialization Rule to each word. 
% It is defined to apply to node (0, 0) 
\hypertarget{top-down-strategy}{}\begin{itemize}
\item[(66)] 
\index{Top-Down Strategy}\textbf{Top-Down Strategy}
\begin{quote}{\ttfamily \raggedright \noindent
\small
Create~an~empty~chart~spanning~the~sentence.~\\
Apply~the~Top-Down~Initialization~Rule.~\\
Until~no~more~edges~are~added:~\\
~~Apply~the~Top-Down~Expand~Rule~everywhere~it~applies.~\\
~~Apply~the~Top-Down~Match~Rule~everywhere~it~applies.~\\
~~Apply~the~Fundamental~Rule~everywhere~it~applies.~\\
Return~all~of~the~parse~trees~corresponding~to~the~parse~edges~in~\\
the~chart.
}\end{quote}
\end{itemize}

We encourage you to experiment with the NLTK chart parser demo,
as before, in order to test out the top-down strategy yourself.


%___________________________________________________________________________

\subsection{The Earley Algorithm}
\label{the-earley-algorithm}

The Earley algorithm \cite{earley1970ecf} is a parsing strategy that
resembles the Top-Down Strategy, but deals more efficiently with
matching against the input string. \href{\#earley-terminology}{Table 8.13} shows the
correspondence between the parsing rules introduced above and the
rules used by the Earley algorithm.
\hypertarget{earley-terminology}{}
\leavevmode
\begin{longtable}[c]{|p{0.377\locallinewidth}|p{0.365\locallinewidth}|}
\hline
\textbf{
\textbf{Top-Down}/\textbf{Bottom-Up}
} & \textbf{
\textbf{Earley}
} \\
\hline
\endhead

Top-Down Initialization Rule
Top-Down Expand Rule
 & 
Predictor Rule
 \\
\hline

Top-Down/Bottom-Up Match Rule
 & 
Scanner Rule
 \\
\hline

Fundamental Rule
 & 
Completer Rule
 \\
\hline
\caption{
Terminology for rules in the Earley algorithm
}\end{longtable}

Let's look in more detail at the Scanner Rule. Suppose the chart
contains an incomplete edge with a lexical category \emph{P} immediately after
the dot,  the next word in the input is \emph{w}, \emph{P} is a part-of-speech
label for \emph{w}. Then the Scanner Rule admits a new complete edge in
which \emph{P} dominates \emph{w}. More precisely:
\hypertarget{scanner-rule}{}\begin{itemize}
\item[(67)] 
\index{Scanner Rule}\textbf{Scanner Rule}
\begin{quote}{\ttfamily \raggedright \noindent
\small
If~the~chart~contains~the~incomplete~edge~\\
~~{[}\emph{A}~→~α~•~\emph{P}~β,~(\emph{i},~\emph{j}){]}~\\
and~~\emph{w}$_{j}$~is~the~\emph{j}$^{th}$~word~of~the~input,~\\
and~\emph{P}~is~a~valid~part~of~speech~for~\emph{w}$_{j}$,~\\
then~add~the~new~complete~edges~\\
~~{[}\emph{P}~→~\emph{w}$_{j}$~•,~(\emph{j},~\emph{j}+1){]}~\\
~~{[}\emph{w}$_{j}$~→~•,~(\emph{j},~\emph{j}+1){]}
}\end{quote}
\end{itemize}

{\noindent} To illustrate, suppose the input is of the form
\docutilsroleexample{I saw ...}, and the chart already contains the edge
{[}\docutilsrolecategory{vp} →  • \docutilsrolecategory{v} ..., (1, 1){]}. Then the Scanner Rule will add to
the chart the edges {[}\docutilsrolecategory{v} -{\textgreater} 'saw', (1, 2){]}
and {[}'saw'→ •, (1, 2){]}. So in effect the Scanner Rule packages up a
sequence of three rule applications: the Bottom-Up Initialization Rule for
{[}\emph{w} → •, (\emph{j}, \emph{j}+1){]},
the Top-Down Expand Rule for {[}\emph{P} → • \emph{w}$_{j}$, (\emph{j}, \emph{j}){]}, and
the Fundamental Rule for {[}\emph{P} → \emph{w}$_{j}$ •, (\emph{j},
\emph{j}+1)){]}. This is considerably more efficient than the Top-Down Strategy, that
adds a new edge of the form {[}\emph{P} → • \emph{w} , (\emph{j}, \emph{j}){]} for
\docutilsroleemphasis{every} lexical rule \emph{P} → \emph{w}, regardless of whether \emph{w} can
be found in the input.
By contrast with Bottom-Up Initialization, however, the
Earley algorithm proceeds strictly left-to-right through the input,
applying all applicable rules at that point in the chart, and never backtracking.
The NLTK chart parser demo, described above, allows the option of
parsing according to the Earley algorithm.


%___________________________________________________________________________

\subsection{Chart Parsing in NLTK}
\label{chart-parsing-in-nltk}

NLTK defines a simple yet flexible chart parser,
\texttt{ChartParser}.  A new chart parser is constructed from a
grammar and a list of chart rules (also known as a \emph{strategy}).  These
rules will be applied, in order, until no new edges are added to the
chart.  In particular, \texttt{ChartParser} uses the algorithm shown in \href{\#chart-algorithm}{(68)}.
\hypertarget{chart-algorithm}{}\begin{itemize}
\item[(68)] \begin{quote}{\ttfamily \raggedright \noindent
\small
Until~no~new~edges~are~added:~\\
~~For~each~chart~rule~\emph{R}:~\\
~~~~Apply~\emph{R}~to~any~applicable~edges~in~the~chart.~\\
~~~~Return~any~complete~parses~in~the~chart.
}\end{quote}
\end{itemize}

\texttt{nltk{\linebreak[0]}.{\linebreak[0]}parse{\linebreak[0]}.{\linebreak[0]}chart} defines two ready-made strategies:
\texttt{TD{\_}STRATEGY}, a basic top-down strategy; and \texttt{BU{\_}STRATEGY}, a
basic bottom-up strategy.  When constructing a chart parser, you
can use either of these strategies, or create your own.

The following example illustrates the use of the chart parser.
We start by defining a simple grammar, and tokenizing a sentence.
We make sure it is a list (not an iterator), since
we wish to use the same tokenized sentence several times.
\hypertarget{chart-demo}{}
\begin{pylisting}
\begin{alltt}\small\textbf{grammar = nltk.parse{\_}cfg(\pysrcstring{'{}'{}'}
\pysrcstring{  NP  -> NNS | JJ NNS | NP CC NP}
\pysrcstring{  NNS -> "men" | "women" | "children" | NNS CC NNS}
\pysrcstring{  JJ  -> "old" | "young"}
\pysrcstring{  CC  -> "and" | "or"}
\pysrcstring{  '{}'{}'})
parser = nltk.ChartParser(grammar, nltk.parse.BU{\_}STRATEGY)}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = \pysrcstring{'old men and women'}.split()
\pysrcprompt{>{}>{}> }\pysrckeyword{for} tree \pysrckeyword{in} parser.nbest{\_}parse(sent):
\pysrcmore{... }    \pysrckeyword{print} tree
\pysrcoutput{(NP (JJ old) (NNS (NNS men) (CC and) (NNS women)))}
\pysrcoutput{(NP (NP (JJ old) (NNS men)) (CC and) (NP (NNS women)))}}\end{alltt}
\caption{Chart Parsing with NLTK}
\vspace{1ex}
\hrule
\end{pylisting}

The \texttt{trace} parameter can be specified when creating a parser, to
turn on tracing (higher trace levels produce more verbose output).
\href{\#butrace}{Example 8.3} shows the trace output for parsing a
sentence with the bottom-up strategy.
Notice that in this output, \texttt{\pysrcstring{'{\linebreak[0]}{[}{\linebreak[0]}-{\linebreak[0]}-{\linebreak[0]}-{\linebreak[0]}-{\linebreak[0]}-{\linebreak[0]}{]}{\linebreak[0]}'}} indicates a complete edge,
\texttt{\pysrcstring{'{\linebreak[0]}>{\linebreak[0]}'}} indicates a self-loop edge, and \texttt{\pysrcstring{'{\linebreak[0]}{[}{\linebreak[0]}-{\linebreak[0]}-{\linebreak[0]}-{\linebreak[0]}-{\linebreak[0]}-{\linebreak[0]}>{\linebreak[0]}'}} indicates an
incomplete edge.
\hypertarget{butrace}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }parser = nltk.ChartParser(grammar, nltk.parse.BU{\_}STRATEGY, trace=2)
\pysrcprompt{>{}>{}> }trees = parser.nbest{\_}parse(sent)
\pysrcoutput{|.   old   .   men   .   and   .  women  .|}
\pysrcoutput{Bottom Up Init Rule:}
\pysrcoutput{|{[}-{}-{}-{}-{}-{}-{}-{}-{}-{]}         .         .         .| {[}0:1{]} 'old'}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{]}         .         .| {[}1:2{]} 'men'}
\pysrcoutput{|.         .         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{]}         .| {[}2:3{]} 'and'}
\pysrcoutput{|.         .         .         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{]}| {[}3:4{]} 'women'}
\pysrcoutput{Bottom Up Predict Rule:}
\pysrcoutput{|>         .         .         .         .| {[}0:0{]} JJ -> * 'old'}
\pysrcoutput{|.         >         .         .         .| {[}1:1{]} NNS -> * 'men'}
\pysrcoutput{|.         .         >         .         .| {[}2:2{]} CC -> * 'and'}
\pysrcoutput{|.         .         .         >         .| {[}3:3{]} NNS -> * 'women'}
\pysrcoutput{Fundamental Rule:}
\pysrcoutput{|{[}-{}-{}-{}-{}-{}-{}-{}-{}-{]}         .         .         .| {[}0:1{]} JJ -> 'old' *}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{]}         .         .| {[}1:2{]} NNS -> 'men' *}
\pysrcoutput{|.         .         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{]}         .| {[}2:3{]} CC -> 'and' *}
\pysrcoutput{|.         .         .         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{]}| {[}3:4{]} NNS -> 'women' *}
\pysrcoutput{Bottom Up Predict Rule:}
\pysrcoutput{|>         .         .         .         .| {[}0:0{]} NP -> * JJ NNS}
\pysrcoutput{|.         >         .         .         .| {[}1:1{]} NP -> * NNS}
\pysrcoutput{|.         >         .         .         .| {[}1:1{]} NNS -> * NNS CC NNS}
\pysrcoutput{|.         .         .         >         .| {[}3:3{]} NP -> * NNS}
\pysrcoutput{|.         .         .         >         .| {[}3:3{]} NNS -> * NNS CC NNS}
\pysrcoutput{Fundamental Rule:}
\pysrcoutput{|{[}-{}-{}-{}-{}-{}-{}-{}-{}->         .         .         .| {[}0:1{]} NP -> JJ * NNS}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{]}         .         .| {[}1:2{]} NP -> NNS *}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}->         .         .| {[}1:2{]} NNS -> NNS * CC NNS}
\pysrcoutput{|{[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{]}         .         .| {[}0:2{]} NP -> JJ NNS *}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}->         .| {[}1:3{]} NNS -> NNS CC * NNS}
\pysrcoutput{|.         .         .         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{]}| {[}3:4{]} NP -> NNS *}
\pysrcoutput{|.         .         .         {[}-{}-{}-{}-{}-{}-{}-{}-{}->| {[}3:4{]} NNS -> NNS * CC NNS}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{]}| {[}1:4{]} NNS -> NNS CC NNS *}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{]}| {[}1:4{]} NP -> NNS *}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}->| {[}1:4{]} NNS -> NNS * CC NNS}
\pysrcoutput{|{[}======================================={]}| {[}0:4{]} NP -> JJ NNS *}
\pysrcoutput{Bottom Up Predict Rule:}
\pysrcoutput{|.         >         .         .         .| {[}1:1{]} NP -> * NP CC NP}
\pysrcoutput{|>         .         .         .         .| {[}0:0{]} NP -> * NP CC NP}
\pysrcoutput{|.         .         .         >         .| {[}3:3{]} NP -> * NP CC NP}
\pysrcoutput{Fundamental Rule:}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}->         .         .| {[}1:2{]} NP -> NP * CC NP}
\pysrcoutput{|{[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}->         .         .| {[}0:2{]} NP -> NP * CC NP}
\pysrcoutput{|.         .         .         {[}-{}-{}-{}-{}-{}-{}-{}-{}->| {[}3:4{]} NP -> NP * CC NP}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}->| {[}1:4{]} NP -> NP * CC NP}
\pysrcoutput{|{[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}->| {[}0:4{]} NP -> NP * CC NP}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}->         .| {[}1:3{]} NP -> NP CC * NP}
\pysrcoutput{|{[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}->         .| {[}0:3{]} NP -> NP CC * NP}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{]}| {[}1:4{]} NP -> NP CC NP *}
\pysrcoutput{|{[}======================================={]}| {[}0:4{]} NP -> NP CC NP *}
\pysrcoutput{|.         {[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}->| {[}1:4{]} NP -> NP * CC NP}
\pysrcoutput{|{[}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}->| {[}0:4{]} NP -> NP * CC NP}}\end{alltt}
\caption{Trace of Bottom-Up Parser}
\vspace{1ex}
\hrule
\end{pylisting}


%___________________________________________________________________________

\subsection{Exercises}
\label{id88}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Use the graphical chart-parser interface to experiment with
different rule invocation strategies. Come up with your own strategy
that you can execute manually using the graphical interface. Describe
the steps, and report any efficiency improvements it has (e.g. in terms
of the size of the resulting chart). Do these improvements depend on
the structure of the grammar? What do you think of the prospects for
significant performance boosts from cleverer rule invocation
strategies?

\item {} 
☼ We have seen that a chart parser adds but never removes edges
from a chart.  Why?

\item {} 
◑ Write a program to compare the efficiency of a top-down chart parser
compared with a recursive descent parser (\href{\#sec-recursive-descent}{Section 7.5.1}).
Use the same grammar and input sentences for both.  Compare their performance
using the \texttt{timeit} module (\href{\#sec-timing}{Section 5.5.4}).

\end{list}


%___________________________________________________________________________

\section{Probabilistic Parsing}
\label{probabilistic-parsing}
% TODO: mention interest in having weights is because they can be learned.
% Without this it is mysterious why we would want to bother.
% Technical aspects follow, but this is important motivation (Steven) 
% TODO: expand opening discussion to remind reader why we're doing this
% (we had moved stuff to front of chapter) (Steven) 

As we pointed out in the introduction to this
chapter, dealing with ambiguity is a key challenge to broad coverage
parsers. We have shown how chart parsing can help improve the
efficiency of computing multiple parses of the same sentences. But
the sheer number of parses can be just overwhelming. We will show how
probabilistic parsing helps to manage a large space of parses.
However, before we deal with these parsing issues,
we must first back up and introduce weighted grammars.


%___________________________________________________________________________

\subsection{Weighted Grammars}
\label{weighted-grammars}

We begin by considering the verb \docutilsroleexample{give}.  This verb requires both
a direct object (the thing being given) and an indirect object (the
recipient).  These complements can be given in either order, as
illustrated in \href{\#dative}{example (69b)}.  In the ``prepositional dative'' form,
the indirect object appears last, and inside a prepositional phrase,
while in the ``double object'' form, the indirect object comes first:
\hypertarget{dative}{}\begin{itemize}
\item[(69)] \begin{itemize}
\item[a.] 
Kim gave a bone to the dog
\end{itemize}
\begin{itemize}
\item[b.] 
Kim gave the dog a bone
\end{itemize}
\end{itemize}

Using the Penn Treebank sample, we can examine all instances of
prepositional dative and double object constructions involving
\docutilsroleexample{give}, as shown in \href{\#give}{Listing 8.4}.
\hypertarget{give}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{give}(t):
    return t.node == \pysrcstring{'VP'} \pysrckeyword{and} len(t) > 2 \pysrckeyword{and} t{[}1{]}.node == \pysrcstring{'NP'}{\textbackslash}
           \pysrckeyword{and} (t{[}2{]}.node == \pysrcstring{'PP-DTV'} \pysrckeyword{or} t{[}2{]}.node == \pysrcstring{'NP'}){\textbackslash}
           \pysrckeyword{and} (\pysrcstring{'give'} \pysrckeyword{in} t{[}0{]}.leaves() \pysrckeyword{or} \pysrcstring{'gave'} \pysrckeyword{in} t{[}0{]}.leaves())
\pysrckeyword{def} \pysrcdefname{sent}(t):
    return \pysrcstring{' '}.join(token \pysrckeyword{for} token \pysrckeyword{in} t.leaves() \pysrckeyword{if} token{[}0{]} \pysrckeyword{not} \pysrckeyword{in} \pysrcstring{'*-0'})
\pysrckeyword{def} \pysrcdefname{print{\_}node}(t, width):
        output = \pysrcstring{"{\%}s {\%}s: {\%}s / {\%}s: {\%}s"} {\%}{\textbackslash}
            (sent(t{[}0{]}), t{[}1{]}.node, sent(t{[}1{]}), t{[}2{]}.node, sent(t{[}2{]}))
        \pysrckeyword{if} len(output) > width:
            output = output{[}:width{]} + \pysrcstring{"..."}
        \pysrckeyword{print} output}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} tree \pysrckeyword{in} nltk.corpus.treebank.parsed{\_}sents():
\pysrcmore{... }    \pysrckeyword{for} t \pysrckeyword{in} tree.subtrees(give):
\pysrcmore{... }        print{\_}node(t, 72)
\pysrcoutput{gave NP: the chefs / NP: a standing ovation}
\pysrcoutput{give NP: advertisers / NP: discounts for maintaining or increasing ad sp...}
\pysrcoutput{give NP: it / PP-DTV: to the politicians}
\pysrcoutput{gave NP: them / NP: similar help}
\pysrcoutput{give NP: them / NP:}
\pysrcoutput{give NP: only French history questions / PP-DTV: to students in a Europe...}
\pysrcoutput{give NP: federal judges / NP: a raise}
\pysrcoutput{give NP: consumers / NP: the straight scoop on the U.S. waste crisis}
\pysrcoutput{gave NP: Mitsui / NP: access to a high-tech medical product}
\pysrcoutput{give NP: Mitsubishi / NP: a window on the U.S. glass industry}
\pysrcoutput{give NP: much thought / PP-DTV: to the rates she was receiving , nor to ...}
\pysrcoutput{give NP: your Foster Savings Institution / NP: the gift of hope and free...}
\pysrcoutput{give NP: market operators / NP: the authority to suspend trading in futu...}
\pysrcoutput{gave NP: quick approval / PP-DTV: to {\$} 3.18 billion in supplemental appr...}
\pysrcoutput{give NP: the Transportation Department / NP: up to 50 days to review any...}
\pysrcoutput{give NP: the president / NP: such power}
\pysrcoutput{give NP: me / NP: the heebie-jeebies}
\pysrcoutput{give NP: holders / NP: the right , but not the obligation , to buy a cal...}
\pysrcoutput{gave NP: Mr. Thomas / NP: only a `{}` qualified '{}' rating , rather than `{}`...}
\pysrcoutput{give NP: the president / NP: line-item veto power}}\end{alltt}
\caption{Usage of Give and Gave in the Penn Treebank sample}
\vspace{1ex}
\hrule
\end{pylisting}

We can observe a strong tendency for the shortest complement to appear
first.  However, this does not account for a form like
\texttt{give{\linebreak[0]} {\linebreak[0]}NP{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}federal{\linebreak[0]} {\linebreak[0]}judges{\linebreak[0]} {\linebreak[0]}/{\linebreak[0]} {\linebreak[0]}NP{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}a{\linebreak[0]} {\linebreak[0]}raise}, where animacy may be
playing a role.  In fact there turn out to be a large number of contributing
factors, as surveyed by \cite{bresnan2006gg}.

How can such tendencies be expressed in a conventional context free
grammar?  It turns out that they cannot.  However, we can address the
problem by adding weights, or probabilities, to the productions of a grammar.

A \index{probabilistic context free grammar}\textbf{probabilistic context free grammar} (or \emph{PCFG}) is a context free
grammar that associates a probability with each of its productions.
It generates the same set of parses for a text that the corresponding
context free grammar does, and assigns a probability to each parse.
The probability of a parse generated by a PCFG is simply the product
of the probabilities of the productions used to generate it.

The simplest way to define a PCFG is to load it from a specially
formatted string consisting of a sequence of weighted productions,
where weights appear in brackets, as shown in \href{\#pcfg1}{Listing 8.5}.
\hypertarget{pcfg1}{}
\begin{pylisting}
\begin{alltt}\small\textbf{grammar = nltk.parse{\_}pcfg(\pysrcstring{"{}"{}"}
\pysrcstring{    S    -> NP VP              {[}1.0{]}}
\pysrcstring{    VP   -> TV NP              {[}0.4{]}}
\pysrcstring{    VP   -> IV                 {[}0.3{]}}
\pysrcstring{    VP   -> DatV NP NP         {[}0.3{]}}
\pysrcstring{    TV   -> 'saw'              {[}1.0{]}}
\pysrcstring{    IV   -> 'ate'              {[}1.0{]}}
\pysrcstring{    DatV -> 'gave'             {[}1.0{]}}
\pysrcstring{    NP   -> 'telescopes'       {[}0.8{]}}
\pysrcstring{    NP   -> 'Jack'             {[}0.2{]}}
\pysrcstring{    "{}"{}"})}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} grammar
\pysrcoutput{Grammar with 9 productions (start state = S)}
\pysrcoutput{    S -> NP VP {[}1.0{]}}
\pysrcoutput{    VP -> TV NP {[}0.4{]}}
\pysrcoutput{    VP -> IV {[}0.3{]}}
\pysrcoutput{    VP -> DatV NP NP {[}0.3{]}}
\pysrcoutput{    TV -> 'saw' {[}1.0{]}}
\pysrcoutput{    IV -> 'ate' {[}1.0{]}}
\pysrcoutput{    DatV -> 'gave' {[}1.0{]}}
\pysrcoutput{    NP -> 'telescopes' {[}0.8{]}}
\pysrcoutput{    NP -> 'Jack' {[}0.2{]}}}\end{alltt}
\caption{Defining a Probabilistic Context Free Grammar (PCFG)}
\vspace{1ex}
\hrule
\end{pylisting}

It is sometimes convenient to combine multiple productions into a single line,
e.g. \texttt{VP{\linebreak[0]} {\linebreak[0]}-{\linebreak[0]}>{\linebreak[0]} {\linebreak[0]}TV{\linebreak[0]} {\linebreak[0]}NP{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}.{\linebreak[0]}4{\linebreak[0]}{]}{\linebreak[0]} {\linebreak[0]}|{\linebreak[0]} {\linebreak[0]}IV{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}.{\linebreak[0]}3{\linebreak[0]}{]}{\linebreak[0]} {\linebreak[0]}|{\linebreak[0]} {\linebreak[0]}DatV{\linebreak[0]} {\linebreak[0]}NP{\linebreak[0]} {\linebreak[0]}NP{\linebreak[0]} {\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}.{\linebreak[0]}3{\linebreak[0]}{]}}.
In order to ensure that the trees generated by the grammar form a
probability distribution, PCFG grammars impose the constraint
that all productions with a given left-hand side must have
probabilities that sum to one.
The grammar in \href{\#pcfg1}{Listing 8.5} obeys this constraint: for \texttt{S},
there is only one production, with a probability of 1.0; for \texttt{VP},
0.4+0.3+0.3=1.0; and for \texttt{NP}, 0.8+0.2=1.0.
The parse tree returned by \texttt{parse{\linebreak[0]}({\linebreak[0]})} includes probabilities:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }viterbi{\_}parser = nltk.ViterbiParser(grammar)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} viterbi{\_}parser.parse({[}\pysrcstring{'Jack'}, \pysrcstring{'saw'}, \pysrcstring{'telescopes'}{]})
\pysrcoutput{(S (NP Jack) (VP (TV saw) (NP telescopes))) (p=0.064)}}\end{alltt}

The next two sections introduce two probabilistic parsing algorithms
for PCFGs.  The first is an A* parser that uses Viterbi-style dynamic
programming to find the single most likely parse for a given text.
Whenever it finds multiple possible parses for a subtree, it discards
all but the most likely parse.  The second is a bottom-up chart parser
that maintains a queue of edges, and adds them to the chart one at a
time.  The ordering of this queue is based on the probabilities
associated with the edges, allowing the parser to expand more likely
edges before less likely ones.  Different queue orderings are used to
implement a variety of different search strategies.  These algorithms
are implemented in the \texttt{nltk{\linebreak[0]}.{\linebreak[0]}parse{\linebreak[0]}.{\linebreak[0]}viterbi} and
\texttt{nltk{\linebreak[0]}.{\linebreak[0]}parse{\linebreak[0]}.{\linebreak[0]}pchart} modules.


%___________________________________________________________________________

\subsection{A* Parser}
\label{a-parser}

An \index{A* Parser}\textbf{A* Parser} is a bottom-up PCFG parser that uses
dynamic programming to find the single most likely parse for a text \cite{klein2003astar}.
It parses texts by iteratively filling in a
\index{most likely constituents table}\textbf{most likely constituents table}.  This table records the most likely tree for each
span and node value.  For example, after parsing the sentence ``I saw the man with
the telescope'' with the grammar \texttt{cfg{\linebreak[0]}.{\linebreak[0]}toy{\_}pcfg1}, the most likely constituents table
contains the following entries (amongst others):
\hypertarget{mlct}{}
\leavevmode
\begin{longtable}[c]{|p{0.066\locallinewidth}|p{0.056\locallinewidth}|p{0.690\locallinewidth}|p{0.138\locallinewidth}|}
\hline
\textbf{
Span
} & \textbf{
Node
} & \textbf{
Tree
} & \textbf{
Prob
} \\
\hline
\endhead

{[}0:1{]}
 & 
NP
 & 
(NP I)
 & 
0.15
 \\
\hline

{[}6:7{]}
 & 
NP
 & 
(NN telescope)
 & 
0.5
 \\
\hline

{[}5:7{]}
 & 
NP
 & 
(NP the telescope)
 & 
0.2
 \\
\hline

{[}4:7{]}
 & 
PP
 & 
(PP with (NP the telescope))
 & 
0.122
 \\
\hline

{[}0:4{]}
 & 
S
 & 
(S (NP I) (VP saw (NP the man)))
 & 
0.01365
 \\
\hline

{[}0:7{]}
 & 
S
 & 
(S (NP I) (VP saw (NP (NP the man) (PP with (NP the telescope)))))
 & 
0.0004163250
 \\
\hline
\caption{
Fragment of Most Likely Constituents Table
}\end{longtable}

Once the table has been completed, the parser
returns the entry for the most likely constituent that spans the
entire text, and whose node value is the start symbol.  For this
example, it would return the entry with a span of {[}0:6{]} and a node
value of ``S''.

Note that we only record the \emph{most likely} constituent for any given
span and node value.  For example, in the table above, there are
actually two possible constituents that cover the span {[}1:6{]} and have
``VP'' node values.
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
``saw the man, who has the telescope'':

\end{list}
\begin{quote}
\begin{description}
\item[{(VP saw}] \leavevmode \begin{description}
\item[{(NP (NP John)}] \leavevmode 
(PP with (NP the telescope))))

\end{description}

\end{description}
\end{quote}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
``used the telescope to see the man'':

\end{list}
\begin{quote}
\begin{description}
\item[{(VP saw}] \leavevmode 
(NP John)
(PP with (NP the telescope)))

\end{description}
\end{quote}

{\noindent}
Since the grammar we are using to parse the text indicates that the
first of these tree structures has a higher probability, the parser
discards the second one.

\textbf{Filling in the Most Likely Constituents Table:}
Because the grammar used by \texttt{ViterbiParse} is a PCFG, the
probability of each constituent can be calculated from the
probabilities of its children.  Since a constituent's children can
never cover a larger span than the constituent itself, each entry of
the most likely constituents table depends only on entries for
constituents with \emph{shorter} spans (or equal spans, in the case of
unary and epsilon productions).

\texttt{ViterbiParse} takes advantage of this fact, and fills in the most
likely constituent table incrementally.  It starts by filling in the
entries for all constituents that span a single element of text.
After it has filled in all the table entries for constituents that
span one element of text, it fills in the entries for constituents
that span two elements of text.  It continues filling in the entries
for constituents spanning larger and larger portions of the text,
until the entire table has been filled.

To find the most likely constituent with a given span and node value,
\texttt{ViterbiParse} considers all productions that could produce that
node value.  For each production, it checks the most likely
constituents table for sequences of children that collectively cover
the span and that have the node values specified by the production's
right hand side.  If the tree formed by applying the production to the
children has a higher probability than the current table entry, then
it updates the most likely constituents table with the new tree.

\textbf{Handling Unary Productions and Epsilon Productions:}
A minor difficulty is introduced by unary productions and epsilon
productions: an entry of the most likely constituents table might
depend on another entry with the same span.  For example, if the
grammar contains the production \texttt{V} → \texttt{VP}, then the table
entries for \texttt{VP} depend on the entries for \texttt{V} with the same span.
This can be a problem if the constituents are checked in the wrong
order.  For example, if the parser tries to find the most likely
constituent for a \texttt{VP} spanning {[}1:3{]} before it finds the most
likely constituents for \texttt{V} spanning {[}1:3{]}, then it can't apply the
\texttt{V} → \texttt{VP} production.

To solve this problem, \texttt{ViterbiParse} repeatedly checks each span
until it finds no new table entries.  Note that cyclic grammar
productions (e.g. \texttt{V} → \texttt{V}) will \emph{not} cause this procedure
to enter an infinite loop.  Since all production probabilities are
less than or equal to 1, any constituent generated by a cycle in the
grammar will have a probability that is less than or equal to the
original constituent; so \texttt{ViterbiParse} will discard it.

In NLTK, we create Viterbi parsers using \texttt{ViterbiParse{\linebreak[0]}({\linebreak[0]})}.
Note that since \texttt{ViterbiParse} only finds the single most likely
parse, that \texttt{nbest{\_}parse{\linebreak[0]}({\linebreak[0]})} will never return more than one parse.
\hypertarget{viterbi-parse}{}
\begin{pylisting}
\begin{alltt}\small\textbf{grammar = nltk.parse{\_}pcfg(\pysrcstring{'{}'{}'}
\pysrcstring{  NP  -> NNS {[}0.5{]} | JJ NNS {[}0.3{]} | NP CC NP {[}0.2{]}}
\pysrcstring{  NNS -> "men" {[}0.1{]} | "women" {[}0.2{]} | "children" {[}0.3{]} | NNS CC NNS {[}0.4{]}}
\pysrcstring{  JJ  -> "old" {[}0.4{]} | "young" {[}0.6{]}}
\pysrcstring{  CC  -> "and" {[}0.9{]} | "or" {[}0.1{]}}
\pysrcstring{  '{}'{}'})
viterbi{\_}parser = nltk.ViterbiParser(grammar)}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sent = \pysrcstring{'old men and women'}.split()
\pysrcprompt{>{}>{}> }\pysrckeyword{print} viterbi{\_}parser.parse(sent)
\pysrcoutput{(NP (JJ old) (NNS (NNS men) (CC and) (NNS women))) (p=0.000864)}}\end{alltt}
\caption{}
\vspace{1ex}
\hrule
\end{pylisting}

The \texttt{trace} method can be used to set the level of tracing output
that is generated when parsing a text.  Trace output displays the
constituents that are considered, and indicates which ones are added
to the most likely constituent table.  It also indicates the
likelihood for each constituent.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }viterbi{\_}parser.trace(3)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} viterbi{\_}parser.parse(sent)
\pysrcoutput{Inserting tokens into the most likely constituents table...}
\pysrcoutput{   Insert: |=...| old}
\pysrcoutput{   Insert: |.=..| men}
\pysrcoutput{   Insert: |..=.| and}
\pysrcoutput{   Insert: |...=| women}
\pysrcoutput{Finding the most likely constituents spanning 1 text elements...}
\pysrcoutput{   Insert: |=...| JJ -> 'old' {[}0.4{]}                 0.4000000000}
\pysrcoutput{   Insert: |.=..| NNS -> 'men' {[}0.1{]}                0.1000000000}
\pysrcoutput{   Insert: |.=..| NP -> NNS {[}0.5{]}                   0.0500000000}
\pysrcoutput{   Insert: |..=.| CC -> 'and' {[}0.9{]}                 0.9000000000}
\pysrcoutput{   Insert: |...=| NNS -> 'women' {[}0.2{]}              0.2000000000}
\pysrcoutput{   Insert: |...=| NP -> NNS {[}0.5{]}                   0.1000000000}
\pysrcoutput{Finding the most likely constituents spanning 2 text elements...}
\pysrcoutput{   Insert: |==..| NP -> JJ NNS {[}0.3{]}                0.0120000000}
\pysrcoutput{Finding the most likely constituents spanning 3 text elements...}
\pysrcoutput{   Insert: |.===| NP -> NP CC NP {[}0.2{]}              0.0009000000}
\pysrcoutput{   Insert: |.===| NNS -> NNS CC NNS {[}0.4{]}           0.0072000000}
\pysrcoutput{   Insert: |.===| NP -> NNS {[}0.5{]}                   0.0036000000}
\pysrcoutput{  Discard: |.===| NP -> NP CC NP {[}0.2{]}              0.0009000000}
\pysrcoutput{  Discard: |.===| NP -> NP CC NP {[}0.2{]}              0.0009000000}
\pysrcoutput{Finding the most likely constituents spanning 4 text elements...}
\pysrcoutput{   Insert: |====| NP -> JJ NNS {[}0.3{]}                0.0008640000}
\pysrcoutput{  Discard: |====| NP -> NP CC NP {[}0.2{]}              0.0002160000}
\pysrcoutput{  Discard: |====| NP -> NP CC NP {[}0.2{]}              0.0002160000}
\pysrcoutput{(NP (JJ old) (NNS (NNS men) (CC and) (NNS women))) (p=0.000864)}}\end{alltt}


%___________________________________________________________________________

\subsection{A Bottom-Up PCFG Chart Parser}
\label{a-bottom-up-pcfg-chart-parser}

The \index{A* parser}\textit{A* parser} described in the previous section finds
the single most likely parse for a given text.  However, when parsers
are used in the context of a larger NLP system, it is often necessary
to produce several alternative parses.  In the context of an overall system,
a parse that is assigned low probability by the parser might still have the
best overall probability.

For example, a probabilistic parser might decide that the most likely
parse for ``I saw John with the cookie'' is is the structure with the
interpretation ``I used my cookie to see John''; but that parse would be
assigned a low probability by a semantic system.  Combining the
probability estimates from the parser and the semantic system, the
parse with the interpretation ``I saw John, who had my cookie'' would be
given a higher overall probability.

This section describes a probabilistic bottom-up chart parser.
It maintains an \index{edge queue}\textbf{edge queue}, and adds these edges to the chart one at a time.
The ordering of this queue is based on the probabilities associated with the edges,
and this allows the parser to insert the most probable edges first.
Each time an edge is added to the chart, it may become possible
to insert new edges, so these are added to the queue.
The bottom-up chart parser continues adding the edges in the
queue to the chart until enough complete parses have been found, or
until the edge queue is empty.

Like an edge in a regular chart, a probabilistic edge
consists of a dotted production, a span, and a (partial) parse tree.
However, unlike ordinary charts, this time the tree is weighted
with a probability.  Its probability
is the product of the probability of the production that
generated it and the probabilities of its children.  For example, the
probability of the edge \texttt{{[}{\linebreak[0]}Edge{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}S} → \texttt{NP} • \texttt{VP{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}0{\linebreak[0]}:{\linebreak[0]}2{\linebreak[0]}{]}}
is the probability of the PCFG production \texttt{S} → \texttt{NP{\linebreak[0]} {\linebreak[0]}VP}
multiplied by the probability of its \docutilsrolecategory{np} child.
(Note that an edge's tree only includes children for elements to the left
of the edge's dot.  Thus, the edge's probability does \emph{not} include
probabilities for the constituents to the right of the edge's dot.)


%___________________________________________________________________________

\subsection{Bottom-Up PCFG Strategies}
\label{bottom-up-pcfg-strategies}

The \index{edge queue}\textit{edge queue} is a sorted list of edges that can be added to the
chart.  It is initialized with a single edge for each token in the
text, with the form \texttt{{[}{\linebreak[0]}Edge{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}token{\linebreak[0]} {\linebreak[0]}|{\linebreak[0]}rarr{\linebreak[0]}|{\linebreak[0]} {\linebreak[0]}|{\linebreak[0]}dot{\linebreak[0]}|{\linebreak[0]}{]}}.
As each edge from the queue is added to the chart, it may
become possible to add further edges, according to two rules:
(i) the Bottom-Up Initialization Rule can be used to add a
self-loop edge whenever an edge whose dot is in position 0 is added to the chart; or
(ii) the Fundamental Rule can be used to combine a new edge
with edges already present in the chart.  These additional edges
are queued for addition to the chart.

By changing the sort order used by the queue, we can control the
strategy that the parser uses to explore the search space.  Since
there are a wide variety of reasonable search strategies,
\texttt{BottomUpChartParser{\linebreak[0]}({\linebreak[0]})} does not define any sort order.
Instead, different strategies are implemented in subclasses of \texttt{BottomUpChartParser{\linebreak[0]}({\linebreak[0]})}.
% We should either explain "inside probabilities" or rename this parser (to
% ``LowestCostFirstParser``?). 

\textbf{Lowest Cost First:}
The simplest way to order the edge queue is to sort edges by the
probabilities of their associated trees (\texttt{nltk{\linebreak[0]}.{\linebreak[0]}InsideChartParser{\linebreak[0]}({\linebreak[0]})}).
This ordering concentrates the efforts of the parser on those edges
that are more likely to be correct analyses of their underlying tokens.

The probability of an edge's tree provides an upper bound on the
probability of any parse produced using that edge.  The probabilistic
``cost'' of using an edge to form a parse is one minus its tree's
probability.  Thus, inserting the edges with the most likely trees
first results in a \index{lowest-cost-first search strategy}\textbf{lowest-cost-first search strategy}.
Lowest-cost-first search is optimal: the first
solution it finds is guaranteed to be the best solution.

However, lowest-cost-first search can be rather inefficient.  Recall that a
tree's probability is the product of the probabilities of all the
productions used to generate it.  Consequently, smaller trees tend to have higher
probabilities than larger ones.  Thus, lowest-cost-first search tends to work
with edges having small trees before considering edges with larger trees.
Yet any complete parse of the text will necessarily have a
large tree, and so this strategy will tend to produce complete parses only
once most other edges are processed.

Let's consider this problem from another angle.
The basic shortcoming with lowest-cost-first search is that it ignores the
probability that an edge's tree will be part of a complete parse.  The parser will
try parses that are locally coherent even if they are unlikely to
form part of a complete parse.  Unfortunately, it can be quite
difficult to calculate the probability that a tree is part of a
complete parse.  However, we can use a variety of techniques to
approximate that probability.

\textbf{Best-First Search:}
This method sorts the edge queue in descending order of the edges'
span, no the assumption that edges having a larger span are more likely
to form part of a complete parse.
Thus, \texttt{LongestParse} employs a \index{best-first search strategy}\textbf{best-first search strategy},
where it inserts the edges that are closest to producing
complete parses before trying any other edges.  Best-first search is
\emph{not} an optimal search strategy: the first solution it finds is not
guaranteed to be the best solution.  However, it will usually find a
complete parse much more quickly than lowest-cost-first search.

\textbf{Beam Search:}
When large grammars are used to parse a text, the edge queue can grow
quite long.  The edges at the end of a large well-sorted queue are
unlikely to be used.  Therefore, it is reasonable to remove (or
\emph{prune}) these edges from the queue.  This strategy is known as
\index{beam search}\textbf{beam search}; it only keeps the best partial results.
The bottom-up chart parsers take an optional parameter \texttt{beam{\_}size};
whenever the edge queue grows longer than this, it is pruned.
This parameter is best used in conjunction with \texttt{InsideChartParser{\linebreak[0]}({\linebreak[0]})}.
Beam search reduces the space requirements for lowest-cost-first
search, by discarding edges that are not likely to be used.  But beam
search also loses many of lowest-cost-first search's more useful
properties.  Beam search is not optimal: it is not guaranteed to find
the best parse first.  In fact, since it might prune a necessary edge,
beam search is not even \index{complete}\textit{complete}: it is not guaranteed to return a
parse if one exists.

In NLTK we can construct these parsers using
\texttt{InsideChartParser}, \texttt{LongestChartParser}, \texttt{RandomChartParser}.
\hypertarget{bottom-up-chart-parsers}{}
\begin{pylisting}
\begin{alltt}\small\textbf{inside{\_}parser = nltk.InsideChartParser(grammar)
longest{\_}parser = nltk.LongestChartParser(grammar)
beam{\_}parser = nltk.InsideChartParser(grammar, beam{\_}size=20)}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} inside{\_}parser.parse(sent)
\pysrcoutput{(NP (JJ old) (NNS (NNS men) (CC and) (NNS women))) (p=0.000864)}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{for} tree \pysrckeyword{in} inside{\_}parser.nbest{\_}parse(sent):
\pysrcmore{... }    \pysrckeyword{print} tree
\pysrcoutput{(NP}
\pysrcoutput{  (JJ old)}
\pysrcoutput{  (NNS (NNS men) (CC and) (NNS women))) (p=0.000864)}
\pysrcoutput{(NP}
\pysrcoutput{  (NP (JJ old) (NNS men))}
\pysrcoutput{  (CC and)}
\pysrcoutput{  (NP (NNS women))) (p=0.000216)}}\end{alltt}
\caption{}
\vspace{1ex}
\hrule
\end{pylisting}

The \texttt{trace} method can be used to set the level of tracing output
that is generated when parsing a text.  Trace output displays edges as
they are added to the chart, and shows the probability for each edges'
tree.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }inside{\_}parser.trace(3)
\pysrcprompt{>{}>{}> }trees = inside{\_}parser.nbest{\_}parse(sent)
\pysrcoutput{  |. . . {[}-{]}| {[}3:4{]} 'women'                          {[}1.0{]}}
\pysrcoutput{  |. . {[}-{]} .| {[}2:3{]} 'and'                            {[}1.0{]}}
\pysrcoutput{  |. {[}-{]} . .| {[}1:2{]} 'men'                            {[}1.0{]}}
\pysrcoutput{  |{[}-{]} . . .| {[}0:1{]} 'old'                            {[}1.0{]}}
\pysrcoutput{  |. . {[}-{]} .| {[}2:3{]} CC -> 'and' *                    {[}0.9{]}}
\pysrcoutput{  |. . > . .| {[}2:2{]} CC -> * 'and'                    {[}0.9{]}}
\pysrcoutput{  |{[}-{]} . . .| {[}0:1{]} JJ -> 'old' *                    {[}0.4{]}}
\pysrcoutput{  |> . . . .| {[}0:0{]} JJ -> * 'old'                    {[}0.4{]}}
\pysrcoutput{  |> . . . .| {[}0:0{]} NP -> * JJ NNS                   {[}0.3{]}}
\pysrcoutput{  |. . . {[}-{]}| {[}3:4{]} NNS -> 'women' *                 {[}0.2{]}}
\pysrcoutput{  |. . . > .| {[}3:3{]} NP -> * NNS                      {[}0.5{]}}
\pysrcoutput{  |. . . > .| {[}3:3{]} NNS -> * NNS CC NNS              {[}0.4{]}}
\pysrcoutput{  |. . . > .| {[}3:3{]} NNS -> * 'women'                 {[}0.2{]}}
\pysrcoutput{  |{[}-> . . .| {[}0:1{]} NP -> JJ * NNS                   {[}0.12{]}}
\pysrcoutput{  |. . . {[}-{]}| {[}3:4{]} NP -> NNS *                      {[}0.1{]}}
\pysrcoutput{  |. . . > .| {[}3:3{]} NP -> * NP CC NP                 {[}0.2{]}}
\pysrcoutput{  |. {[}-{]} . .| {[}1:2{]} NNS -> 'men' *                   {[}0.1{]}}
\pysrcoutput{  |. > . . .| {[}1:1{]} NP -> * NNS                      {[}0.5{]}}
\pysrcoutput{  |. > . . .| {[}1:1{]} NNS -> * NNS CC NNS              {[}0.4{]}}
\pysrcoutput{  |. > . . .| {[}1:1{]} NNS -> * 'men'                   {[}0.1{]}}
\pysrcoutput{  |. . . {[}->| {[}3:4{]} NNS -> NNS * CC NNS              {[}0.08{]}}
\pysrcoutput{  |. {[}-{]} . .| {[}1:2{]} NP -> NNS *                      {[}0.05{]}}
\pysrcoutput{  |. > . . .| {[}1:1{]} NP -> * NP CC NP                 {[}0.2{]}}
\pysrcoutput{  |. {[}-> . .| {[}1:2{]} NNS -> NNS * CC NNS              {[}0.04{]}}
\pysrcoutput{  |. {[}-{}-{}-> .| {[}1:3{]} NNS -> NNS CC * NNS              {[}0.036{]}}
\pysrcoutput{  |. . . {[}->| {[}3:4{]} NP -> NP * CC NP                 {[}0.02{]}}
\pysrcoutput{  |{[}-{}-{}-{]} . .| {[}0:2{]} NP -> JJ NNS *                   {[}0.012{]}}
\pysrcoutput{  |> . . . .| {[}0:0{]} NP -> * NP CC NP                 {[}0.2{]}}
\pysrcoutput{  |. {[}-> . .| {[}1:2{]} NP -> NP * CC NP                 {[}0.01{]}}
\pysrcoutput{  |. {[}-{}-{}-> .| {[}1:3{]} NP -> NP CC * NP                 {[}0.009{]}}
\pysrcoutput{  |. {[}-{}-{}-{}-{}-{]}| {[}1:4{]} NNS -> NNS CC NNS *              {[}0.0072{]}}
\pysrcoutput{  |. {[}-{}-{}-{}-{}-{]}| {[}1:4{]} NP -> NNS *                      {[}0.0036{]}}
\pysrcoutput{  |. {[}-{}-{}-{}-{}->| {[}1:4{]} NNS -> NNS * CC NNS              {[}0.00288{]}}
\pysrcoutput{  |{[}-{}-{}-> . .| {[}0:2{]} NP -> NP * CC NP                 {[}0.0024{]}}
\pysrcoutput{  |{[}-{}-{}-{}-{}-> .| {[}0:3{]} NP -> NP CC * NP                 {[}0.00216{]}}
\pysrcoutput{  |. {[}-{}-{}-{}-{}-{]}| {[}1:4{]} NP -> NP CC NP *                 {[}0.0009{]}}
\pysrcoutput{  |{[}======={]}| {[}0:4{]} NP -> JJ NNS *                   {[}0.000864{]}}
\pysrcoutput{  |. {[}-{}-{}-{}-{}->| {[}1:4{]} NP -> NP * CC NP                 {[}0.00072{]}}
\pysrcoutput{  |{[}======={]}| {[}0:4{]} NP -> NP CC NP *                 {[}0.000216{]}}
\pysrcoutput{  |. {[}-{}-{}-{}-{}->| {[}1:4{]} NP -> NP * CC NP                 {[}0.00018{]}}
\pysrcoutput{  |{[}-{}-{}-{}-{}-{}-{}->| {[}0:4{]} NP -> NP * CC NP                 {[}0.0001728{]}}
\pysrcoutput{  |{[}-{}-{}-{}-{}-{}-{}->| {[}0:4{]} NP -> NP * CC NP                 {[}4.32e-05{]}}}\end{alltt}


%___________________________________________________________________________

\section{Grammar Induction}
\label{grammar-induction}

As we have seen, PCFG productions are just like CFG productions,
adorned with probabilities.  So far, we have simply specified these
probabilities in the grammar.  However, it is more usual to \emph{estimate}
these probabilities from training data, namely a collection of parse
trees or \emph{treebank}.

The simplest method uses \emph{Maximum Likelihood Estimation}, so called
because probabilities are chosen in order to maximize the likelihood
of the training data.  The probability of a production
\texttt{VP} → \texttt{V{\linebreak[0]} {\linebreak[0]}NP{\linebreak[0]} {\linebreak[0]}PP} is \emph{p(V,NP,PP {\textbar} VP)}.  We calculate this as
follows:
\begin{quote}{\ttfamily \raggedright \noindent
\small
~~~~~~~~~~~~~~~~~~count(VP~->~V~NP~PP)~\\
P(V,NP,PP~|~VP)~=~-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-~\\
~~~~~~~~~~~~~~~~~~count(VP~->~...)
}\end{quote}

Here is a simple program that induces a grammar from the first
three parse trees in the Penn Treebank corpus:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} itertools \pysrckeyword{import} islice
\pysrcprompt{>{}>{}> }productions = {[}{]}
\pysrcprompt{>{}>{}> }S = nltk.Nonterminal(\pysrcstring{'S'})
\pysrcprompt{>{}>{}> }\pysrckeyword{for} tree \pysrckeyword{in} nltk.corpus.treebank.parsed{\_}sents(\pysrcstring{'wsj{\_}0002.mrg'}):
\pysrcmore{... }     productions += tree.productions()
\pysrcprompt{>{}>{}> }grammar = nltk.induce{\_}pcfg(S, productions)
\pysrcprompt{>{}>{}> }\pysrckeyword{for} production \pysrckeyword{in} grammar.productions(){[}:10{]}:
\pysrcmore{... }     \pysrckeyword{print} production
\pysrcoutput{CC -> 'and' {[}1.0{]}}
\pysrcoutput{NNP -> 'Agnew' {[}0.166666666667{]}}
\pysrcoutput{JJ -> 'industrial' {[}0.2{]}}
\pysrcoutput{NP -> CD NNS {[}0.142857142857{]}}
\pysrcoutput{, -> ',' {[}1.0{]}}
\pysrcoutput{S -> NP-SBJ NP-PRD {[}0.5{]}}
\pysrcoutput{VP -> VBN S {[}0.5{]}}
\pysrcoutput{NNP -> 'Rudolph' {[}0.166666666667{]}}
\pysrcoutput{NP -> NP PP {[}0.142857142857{]}}
\pysrcoutput{NNP -> 'PLC' {[}0.166666666667{]}}}\end{alltt}
\hypertarget{sec-normal-forms}{}

%___________________________________________________________________________

\subsection{Normal Forms}
\label{normal-forms}
\label{sec-normal-forms}

Grammar induction usually involves normalizing the grammar
in various ways.  NLTK trees
support binarization (Chomsky Normal Form), parent annotation,
Markov order-N smoothing, and unary collapsing:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }treebank{\_}string = \pysrcstring{"{}"{}"(S (NP-SBJ (NP (QP (IN at) (JJS least) (CD nine) (NNS tenths)) )}
\pysrcmore{... }\pysrcstring{    (PP (IN of) (NP (DT the) (NNS students) ))) (VP (VBD passed)))"{}"{}"}
\pysrcprompt{>{}>{}> }t = nltk.bracket{\_}parse(treebank{\_}string)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} t
\pysrcoutput{(S}
\pysrcoutput{  (NP-SBJ}
\pysrcoutput{    (NP (QP (IN at) (JJS least) (CD nine) (NNS tenths)))}
\pysrcoutput{    (PP (IN of) (NP (DT the) (NNS students))))}
\pysrcoutput{  (VP (VBD passed)))}
\pysrcoutput{}\pysrcprompt{>{}>{}> }t.collapse{\_}unary(collapsePOS=True)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} t
\pysrcoutput{(S}
\pysrcoutput{  (NP-SBJ}
\pysrcoutput{    (NP+QP (IN at) (JJS least) (CD nine) (NNS tenths))}
\pysrcoutput{    (PP (IN of) (NP (DT the) (NNS students))))}
\pysrcoutput{  (VP+VBD passed))}
\pysrcoutput{}\pysrcprompt{>{}>{}> }t.chomsky{\_}normal{\_}form()
\pysrcprompt{>{}>{}> }\pysrckeyword{print} t
\pysrcoutput{(S}
\pysrcoutput{  (NP-SBJ}
\pysrcoutput{    (NP+QP}
\pysrcoutput{      (IN at)}
\pysrcoutput{      (NP+QP|<JJS-CD-NNS>}
\pysrcoutput{        (JJS least)}
\pysrcoutput{        (NP+QP|<CD-NNS> (CD nine) (NNS tenths))))}
\pysrcoutput{    (PP (IN of) (NP (DT the) (NNS students))))}
\pysrcoutput{  (VP+VBD passed))}}\end{alltt}

These trees are shown in \href{\#treetransforms}{(70c)}.
\hypertarget{treetransforms}{}\begin{itemize}
\item[(70)] \begin{itemize}
\item[a.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-23.pdf}}}\end{itemize}
\begin{itemize}
\item[b.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-24.pdf}}}\end{itemize}
\begin{itemize}
\item[c.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-25.pdf}}}\end{itemize}
\end{itemize}


%___________________________________________________________________________

\section{Conclusion}
\label{id91}


%___________________________________________________________________________

\section{Further Reading}
\label{id92}

Section 13.4 of \cite{jurafskymartin2008} covers chart parsing, and Chapter 14
contains a more formal presentation of statistical parsing.
\begin{itemize}
\item {} 
\cite{manning2003probabilistic}

\item {} 
\cite{klein2003astar}

\end{itemize}
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}

\part{ADVANCED TOPICS}

\textbf{Introduction to Part III}

Part III covers a selection of advanced topics.
It begins with a chapter on applied programming in Python,
covering topics in program development, standard libraries,
and algorithm design.  The following two chapters focus on
making grammars more expressive, first by adding the powerful
notation of distinctive features, and then by using this to
add semantic interpretation.  The final chapter deals with
linguistic data management.
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
% TODO: explain why a program should not usually import all of NLTK. 
\hypertarget{chap-applied-programming}{}\hypertarget{t9}{}\hypertarget{oo}{}\hypertarget{sec-variable-scope}{}\hypertarget{kitty}{}\hypertarget{brent}{}\hypertarget{sec-sets}{}\hypertarget{set1}{}\hypertarget{set2}{}\hypertarget{set3}{}\hypertarget{set4}{}\hypertarget{compax}{}\hypertarget{comprax}{}\hypertarget{listcomp}{}\hypertarget{modrel}{}\setcounter{chapter}{8}

%___________________________________________________________________________

\chapter{Applied Programming in Python}
\label{applied-programming-in-python}
\label{modrel}
\label{listcomp}
\label{comprax}
\label{compax}
\label{set4}
\label{set3}
\label{set2}
\label{set1}
\label{sec-sets}
\label{brent}
\label{kitty}
\label{sec-variable-scope}
\label{oo}
\label{t9}
\label{chap-applied-programming}

\emph{This chapter is in development.}
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% TODO:
% discuss applications of unify() to dicts, lists, and mixtures of dicts and lists
% FeatureValueTuple: '[x=(1,2,3)]'
% FeatureValueSet: '[x={1,2,3}]'
% NB: Unification does *not* descend into tuples or sets; but variable
% substitution from bindings does.  Generally speaking, tuples and set
% feature values should never contain feature structures.
% FeatureValueUnion: '{?a+?b}', which will automatically collapse to a
% FeatureValueSet as soon as all top-level variables are replaced with set values.
% As with FeatureValueSet, unification does not descend into FeatureValueUnion,
% but variable binding does.
% More examples are in Edward's email of 25 August 
% standard global imports
% 
% >>> import nltk, re, pprint 
\hypertarget{chap-featgram}{}\setcounter{chapter}{9}

%___________________________________________________________________________

\chapter{Feature Based Grammar}
\label{feature-based-grammar}
\label{chap-featgram}


%___________________________________________________________________________

\section{Introduction}
\label{id96}

Imagine you are building a spoken dialogue
system to answer queries about train schedules in Europe. \href{\#train0}{(71)}
illustrates one of the input
sentences that the system should handle.
\hypertarget{train0}{}\begin{itemize}
\item[(71)] 
Which stations does the 9.00 express from Amsterdam to Paris stop at?
\end{itemize}

The information that the customer is seeking is not exotic — the
system back-end just needs to look up the list of stations on the
route, and reel them off. But you have to be careful in giving the
correct semantic interpretation to
\href{\#train0}{(71)}. You don't want to end up with the
system trying to answer \href{\#train1}{(72)} instead:
\hypertarget{train1}{}\begin{itemize}
\item[(72)] 
Which station does the 9.00 express from Amsterdam terminate at?
\end{itemize}

Part of your solution might use domain knowledge to figure out that if
a speaker knows that the train is a train to Paris, then she probably
isn't asking about the terminating station in \href{\#train0}{(71)}. But the
solution will also involve recognizing the syntactic structure of the
speaker's query.  In particular, your analyzer must recognize that
there is a syntactic connection between the question phrase \docutilsroleexample{which
stations}, and the phrase \docutilsroleexample{stop at} at the end \href{\#train0}{(71)}. The
required interpretation is made clearer in the ``quiz question version
shown in \href{\#train2}{(73)}, where the question phrase fills the ''gap`` that is
implicit in \href{\#train0}{(71)}:
\hypertarget{train2}{}\begin{itemize}
\item[(73)] 
The 9.00 express from Amsterdam to Paris stops at which stations?
\end{itemize}

The \index{long-distance dependency}\textbf{long-distance dependency} between an initial question phrase
and the gap that it semantically connects to cannot be recognized by
techniques we have presented in earlier chapters. For example, we
can't use \emph{n}-gram based language models; in practical terms, it is
infeasible to observe the \emph{n}-grams for a big enough value of
\emph{n}. Similarly, chunking grammars only attempt to capture local
patterns, and therefore just don't ''see`` long-distance dependencies.
In this chapter, we will show how syntactic features can be used to
provide a simple yet effective technique for keeping track of
long-distance dependencies in sentences.

Features are helpful too for dealing with purely local dependencies.
Consider the German questions \href{\#germanagr}{(74)}.
\hypertarget{germanagr}{}\begin{itemize}
\item[(74)] \end{itemize}

The only way of telling which noun phrase is the subject of
\docutilsroleexample{kennen} ('know') and which is the object is by looking at the
agreement inflection on the verb — word order is no help to us
here. Since verbs in German agree in number with their subjects, the
plural form \docutilsroleexample{kennen} requires \docutilsroleexample{Welche Studenten} as subject,
while the singular form \docutilsroleexample{kennt} requires \docutilsroleexample{Franz} as
subject. The fact that subjects and verbs must agree in number can be
expressed within the CFGs that we presented in \href{\#chap-parse}{Chapter 7}. But capturing the fact that the interpretations of
\href{..gloss::Welche|Studenten|kennen|Franz?which|student.PL|know.PL|Franz'whichstudentsknowFranz?'}{germanagra} and \href{..gloss::Welche|Studenten|kennt|Franz?which|student.PL|know.SG|Franz'whichstudentsdoesFranzknow?'}{germanagrb} differ is more challenging. In this
chapter, we will only examine the syntactic aspect of local
dependencies such as number agreement. In \href{\#chap-semantics}{Chapter 11}, we
will demonstrate how feature-based grammars can be extended so that
they build a representation of meaning in parallel with a
representation of syntactic structure.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Remember that our program samples assume you
begin your interactive session or your program with: \texttt{\pysrckeyword{import} {\linebreak[0]}nltk{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}re{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}pprint}
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\section{Why Features?}
\label{why-features}

We have already used the term ''feature`` a few times, without saying
what it means. What's special about feature-based grammars? The core
ideas are probably already familiar to you. To make things concrete,
let's look at the simple phrase \titlereference{these dogs}. It's composed of two
words. We'll be a bit abstract for the moment, and call these words
\titlereference{a} and \titlereference{b}. We'll be modest, and assume that we do not know
\docutilsroleemphasis{everything} about them, but we can at least give a partial
description. For example, we know that the orthography of \titlereference{a} is
\docutilsroleexample{these}, its phonological form is \titlereference{DH IY Z},  its part-of-speech is
\docutilsrolecategory{Det}, and its number is plural. We can use dot notation to record
these observations:
\hypertarget{feat0}{}\begin{itemize}
\item[(75)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\titlereference{a.spelling}~=~\docutilsroleexample{these}~\\
\titlereference{a.phonology}~=~\titlereference{DH~IY~Z}~\\
\titlereference{a.pos}~=~\docutilsrolecategory{Det}~\\
\titlereference{a.number}~=~plural
}\end{quote}
\end{itemize}

Thus \href{\#feat0}{(75)} is a \titlereference{partial description} of a word; it lists some
attributes, or features, of the word, and declares their values. There
are other attributes that we might be interested in, which have
not been specified; for example, what head the word is dependent on
(using the notion of dependency discussed in \href{\#chap-parse}{Chapter 7}), and
what the lemma of the word is. But this omission of some attributes is
exactly what you would expect from a partial description!
% The framework of context-free grammars that we presented in Chapter
% chap-parse_ describes syntactic constituents with the help of a
% limited set of category labels. These atomic labels are adequate for
% talking about the gross structure of sentences. But when we start to
% make finer grammatical distinctions it becomes awkward to enrich the
% set of categories in a systematic manner. In this chapter we will
% address this challenge by decomposing categories using features
% (somewhat similar to the key-value pairs of Python dictionaries). 

We will start off this chapter by looking more closely at the
phenomenon of syntactic agreement; we will show how agreement
constraints can be expressed elegantly using features, and illustrate
their use in a simple grammar. Feature structures are a general data
structure for representing information of any kind; we will briefly
look at them from a more formal point of view, and explain how to
create feature structures in Python. In the final part of the chapter,
we demonstrate that the additional expressiveness of features opens
out a wide spectrum of possibilities for describing sophisticated
aspects of linguistic structure.


%___________________________________________________________________________

\subsection{Syntactic Agreement}
\label{syntactic-agreement}

Consider the following contrasts:
\hypertarget{thisdog}{}\begin{itemize}
\item[(76)] \begin{itemize}
\item[a.] 
this dog
\end{itemize}
\begin{itemize}
\item[b.] 
*these dog
\end{itemize}
\end{itemize}
\hypertarget{thesedogs}{}\begin{itemize}
\item[(77)] \begin{itemize}
\item[a.] 
these dogs
\end{itemize}
\begin{itemize}
\item[b.] 
*this dogs
\end{itemize}
\end{itemize}

{\noindent} In English, nouns are usually morphologically marked as being singular
or plural. The form of the demonstrative also varies:
\docutilsroleexample{this} (singular) and \docutilsroleexample{these} (plural).
Examples \href{\#thisdog}{(76b)} and \href{\#thesedogs}{(77b)} show that there are constraints on
the use of demonstratives and nouns within a noun phrase:
either both are singular or both are plural. A similar
constraint holds between subjects and predicates:
\hypertarget{subjpredsg}{}\begin{itemize}
\item[(78)] \begin{itemize}
\item[a.] 
the dog runs
\end{itemize}
\begin{itemize}
\item[b.] 
*the dog run
\end{itemize}
\end{itemize}
\hypertarget{subjpredpl}{}\begin{itemize}
\item[(79)] \begin{itemize}
\item[a.] 
the dogs run
\end{itemize}
\begin{itemize}
\item[b.] 
*the dogs runs
\end{itemize}
\end{itemize}
% Proposed for deletion: The element which determines the
% agreement, here the subject noun phrase, is called the agreement
% `controller`:dt:, while the element whose form is determined by
% agreement, here the verb, is called the `target`:dt:. 

{\noindent} Here we can see that morphological properties of the verb co-vary
with syntactic properties of the subject noun phrase.  This co-variance is
called \index{agreement}\textbf{agreement}.
If we look further at verb agreement in English, we will see that
present tense verbs typically have two inflected forms: one for third person
singular, and another for every other combination of person and number:
\hypertarget{agreement-paradigm}{}
\leavevmode
\begin{longtable}[c]{|p{0.156\locallinewidth}|p{0.168\locallinewidth}|p{0.133\locallinewidth}|}
\hline
 & 
\textbf{singular}
 & 
\textbf{plural}
 \\
\hline

\textbf{1st per}
 & 
\emph{I run}
 & 
\emph{we run}
 \\
\hline

\textbf{2nd per}
 & 
\emph{you run}
 & 
\emph{you run}
 \\
\hline

\textbf{3rd per}
 & 
\emph{he/she/it
runs}
 & 
\emph{they run}
 \\
\hline
\caption{
Agreement Paradigm for English Regular Verbs
}\end{longtable}

We can make the role of morphological properties a bit more explicit
as illustrated in \href{..gloss::the|dog|run-s|dog.3.SG|run-3.SG}{runs} and \href{..gloss::the|dog-s|run|dog.3.PL|run-3.PL}{run}. These representations indicate that
the verb agrees with its subject in person and number. (We use ''3`` as
an abbreviation for 3rd person, ''SG`` for singular and ''PL`` for plural.)

Let's see what happens when we encode these agreement constraints in a
context-free grammar.  We will begin with the simple CFG in \href{\#agcfg0}{(80)}.
\hypertarget{agcfg0}{}\begin{itemize}
\item[(80)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{s}~→~\docutilsrolecategory{np~vp}~\\
\docutilsrolecategory{np}~→~\docutilsrolecategory{Det~n}~\\
\docutilsrolecategory{vp}~→~\docutilsrolecategory{v}~\\
~\\
\docutilsrolecategory{Det}~→~'this'~\\
\docutilsrolecategory{n}~→~'dog'~\\
\docutilsrolecategory{v}~→~'runs'
}\end{quote}
\end{itemize}

{\noindent} \href{\#agcfg0}{Example (80)} allows us to generate the sentence \docutilsroleexample{this dog runs};
however, what we really want to do is also generate \docutilsroleexample{these dogs
run} while blocking unwanted strings such as \docutilsroleexample{*this dogs run}
and \docutilsroleexample{*these dog runs}. The most straightforward approach is to
add new non-terminals and productions to the grammar:
\hypertarget{agcfg1}{}\begin{itemize}
\item[(81)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{S{\_}SG}~→~\docutilsrolecategory{NP{\_}SG~VP{\_}SG}~\\
\docutilsrolecategory{S{\_}PL}~→~\docutilsrolecategory{NP{\_}PL~VP{\_}PL}~\\
\docutilsrolecategory{NP{\_}SG}~→~\docutilsrolecategory{Det{\_}SG~N{\_}SG}~\\
\docutilsrolecategory{NP{\_}PL}~→~\docutilsrolecategory{Det{\_}PL~N{\_}PL}~\\
\docutilsrolecategory{VP{\_}SG}~→~\docutilsrolecategory{V{\_}SG}~\\
\docutilsrolecategory{VP{\_}PL}~→~\docutilsrolecategory{V{\_}PL}~\\
~\\
\docutilsrolecategory{Det{\_}SG}~→~'this'~\\
\docutilsrolecategory{Det{\_}PL}~→~'these'~\\
\docutilsrolecategory{N{\_}SG}~→~'dog'~\\
\docutilsrolecategory{N{\_}PL}~→~'dogs'~\\
\docutilsrolecategory{V{\_}SG}~→~'runs'~\\
\docutilsrolecategory{V{\_}PL}~→~'run'
}\end{quote}
\end{itemize}

{\noindent} It should be clear that this grammar will do the required
task, but only at the cost of duplicating our previous set of
productions.
% Rule multiplication is of course more severe if we add in
% person agreement constraints.
% "rule multiplication" will be meaningless to some readers.
% We need to be consistent in referring to these as productions. 


%___________________________________________________________________________

\subsection{Using Attributes and Constraints}
\label{using-attributes-and-constraints}

We spoke informally of linguistic categories having \emph{properties}; for
example, that a noun has the property of being plural. Let's
make this explicit:
\hypertarget{num0}{}\begin{itemize}
\item[(82)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{N}{[}\docutilsrolefeature{num}=\docutilsrolefval{pl}{]}
}\end{quote}
\end{itemize}

{\noindent} In \href{\#num0}{(82)}, we have introduced some new notation which says that the
category \docutilsrolecategory{N} has a \index{feature}\textbf{feature} called \docutilsrolefeature{num} (short for
'number') and that the value of this feature is \docutilsrolefval{pl} (short for
'plural'). We can add similar annotations to other categories, and use
them in lexical entries:
\hypertarget{agcfg2}{}\begin{itemize}
\item[(83)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{Det}{[}\docutilsrolefeature{num}=\docutilsrolefval{sg}{]}~→~'this'~\\
\docutilsrolecategory{Det}{[}\docutilsrolefeature{num}=\docutilsrolefval{pl}{]}~~→~'these'~\\
\docutilsrolecategory{N}{[}\docutilsrolefeature{num}=\docutilsrolefval{sg}{]}~→~'dog'~\\
\docutilsrolecategory{N}{[}\docutilsrolefeature{num}=\docutilsrolefval{pl}{]}~→~'dogs'~\\
\docutilsrolecategory{V}{[}\docutilsrolefeature{num}=\docutilsrolefval{sg}{]}~→~'runs'~\\
\docutilsrolecategory{V}{[}\docutilsrolefeature{num}=\docutilsrolefval{pl}{]}~→~'run'
}\end{quote}
\end{itemize}

{\noindent} Does this help at all? So far, it looks just like a slightly more
verbose alternative to what was specified in \href{\#agcfg1}{(81)}. Things become
more interesting when we allow \emph{variables} over feature values, and use
these to state constraints:
\hypertarget{agcfg3}{}\begin{itemize}
\item[(84)] \hypertarget{srule}{}\begin{itemize}
\item[a.] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{S}~→~\docutilsrolecategory{NP}{[}\docutilsrolefeature{num}=\docutilsrolemath{?n}{]}~\docutilsrolecategory{VP}{[}\docutilsrolefeature{num}=\docutilsrolemath{?n}{]}
}\end{quote}
\end{itemize}
\hypertarget{nprule}{}\begin{itemize}
\item[b.] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{NP}{[}\docutilsrolefeature{num}=\docutilsrolemath{?n}{]}~→~\docutilsrolecategory{Det}{[}\docutilsrolefeature{num}=\docutilsrolemath{?n}{]}~\docutilsrolecategory{N}{[}\docutilsrolefeature{num}=\docutilsrolemath{?n}{]}
}\end{quote}
\end{itemize}
\hypertarget{vprule}{}\begin{itemize}
\item[c.] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{VP}{[}\docutilsrolefeature{num}=\docutilsrolemath{?n}{]}~→~\docutilsrolecategory{V}{[}\docutilsrolefeature{num}=\docutilsrolemath{?n}{]}
}\end{quote}
\end{itemize}
\end{itemize}

{\noindent} We are using ''\docutilsrolemath{?n}`` as a variable over values of \docutilsrolefeature{num}; it can
be instantiated either to \docutilsrolefval{sg} or \docutilsrolefval{pl}. Its scope is
limited to individual productions. That is, within \href{\#srule}{(84a)}, for example,
\docutilsrolemath{?n} must be instantiated to the same constant value; we can
read the production as saying that whatever value \docutilsrolecategory{NP} takes for the feature
\docutilsrolefeature{num}, \docutilsrolecategory{VP} must take the same value.

In order to understand how these feature constraints work, it's
helpful to think about how one would go about building a tree. Lexical
productions will admit the following local trees (trees of
depth one):
\begin{itemize}
\item[(85)] \hypertarget{this}{}\begin{itemize}
\item[a.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-26.pdf}}}\end{itemize}
\hypertarget{these}{}\begin{itemize}
\item[b.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-27.pdf}}}\end{itemize}
\end{itemize}
\begin{itemize}
\item[(86)] \hypertarget{dog}{}\begin{itemize}
\item[a.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-28.pdf}}}\end{itemize}
\hypertarget{dogs}{}\begin{itemize}
\item[b.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-29.pdf}}}\end{itemize}
\end{itemize}

{\noindent} Now \href{\#nprule}{(84b)} says that whatever the \docutilsrolefeature{num} values of \docutilsrolecategory{N} and
\docutilsrolecategory{Det} are, they have to be the same. Consequently,  \href{\#nprule}{(84b)} will
permit \href{\#this}{(85a)} and \href{\#dog}{(86a)} to be combined into an \docutilsrolecategory{NP} as shown in
\href{\#good1}{(87a)} and it will also allow \href{\#these}{(85b)} and \href{\#dogs}{(86b)} to be combined, as in
\href{\#good2}{(87b)}. By contrast,  \href{\#bad1}{(88a)} and \href{\#bad2}{(88b)} are prohibited because the roots
of their
constituent local trees differ in their values for the \docutilsrolefeature{num} feature.
\begin{itemize}
\item[(87)] \hypertarget{good1}{}\begin{itemize}
\item[a.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-30.pdf}}}\end{itemize}
\hypertarget{good2}{}\begin{itemize}
\item[b.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-31.pdf}}}\end{itemize}
\end{itemize}
\begin{itemize}
\item[(88)] \hypertarget{bad1}{}\begin{itemize}
\item[a.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-32.pdf}}}\end{itemize}
\hypertarget{bad2}{}\begin{itemize}
\item[b.] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-33.pdf}}}\end{itemize}
\end{itemize}

Production \href{\#vprule}{(84c)} can be thought of as saying that the \docutilsrolefeature{num} value of the
head verb has to be the same as the \docutilsrolefeature{num} value of the \docutilsrolecategory{VP}
mother. Combined with \href{\#srule}{(84a)}, we derive the consequence that if the
\docutilsrolefeature{num} value of the subject head noun is \docutilsrolefval{pl}, then so is
the \docutilsrolefeature{num} value of the \docutilsrolecategory{VP}'s head verb.
\begin{itemize}
\item[(89)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-34.pdf}}}\end{itemize}

The grammar in \href{\#feat0cfg}{listing 10.1} illustrates most of the ideas we have introduced so
far in this chapter, plus a couple of new ones.
\hypertarget{feat0cfg}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.data.show{\_}cfg(\pysrcstring{'grammars/feat0.fcfg'})
\pysrcoutput{{\%} start S}
\pysrcoutput{{\#} {\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}}
\pysrcoutput{{\#} Grammar Rules}
\pysrcoutput{{\#} {\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}}
\pysrcoutput{{\#} S expansion rules}
\pysrcoutput{S -> NP{[}NUM=?n{]} VP{[}NUM=?n{]}}
\pysrcoutput{{\#} NP expansion rules}
\pysrcoutput{NP{[}NUM=?n{]} -> N{[}NUM=?n{]}}
\pysrcoutput{NP{[}NUM=?n{]} -> PropN{[}NUM=?n{]}}
\pysrcoutput{NP{[}NUM=?n{]} -> Det{[}NUM=?n{]} N{[}NUM=?n{]}}
\pysrcoutput{NP{[}NUM=pl{]} -> N{[}NUM=pl{]}}
\pysrcoutput{{\#} VP expansion rules}
\pysrcoutput{VP{[}TENSE=?t, NUM=?n{]} -> IV{[}TENSE=?t, NUM=?n{]}}
\pysrcoutput{VP{[}TENSE=?t, NUM=?n{]} -> TV{[}TENSE=?t, NUM=?n{]} NP}
\pysrcoutput{{\#} {\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}}
\pysrcoutput{{\#} Lexical Rules}
\pysrcoutput{{\#} {\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}}
\pysrcoutput{Det{[}NUM=sg{]} -> 'this' | 'every'}
\pysrcoutput{Det{[}NUM=pl{]} -> 'these' | 'all'}
\pysrcoutput{Det -> 'the' | 'some'}
\pysrcoutput{PropN{[}NUM=sg{]}-> 'Kim' | 'Jody'}
\pysrcoutput{N{[}NUM=sg{]} -> 'dog' | 'girl' | 'car' | 'child'}
\pysrcoutput{N{[}NUM=pl{]} -> 'dogs' | 'girls' | 'cars' | 'children'}
\pysrcoutput{IV{[}TENSE=pres,  NUM=sg{]} -> 'disappears' | 'walks'}
\pysrcoutput{TV{[}TENSE=pres, NUM=sg{]} -> 'sees' | 'likes'}
\pysrcoutput{IV{[}TENSE=pres,  NUM=pl{]} -> 'disappear' | 'walk'}
\pysrcoutput{TV{[}TENSE=pres, NUM=pl{]} -> 'see' | 'like'}
\pysrcoutput{IV{[}TENSE=past, NUM=?n{]} -> 'disappeared' | 'walked'}
\pysrcoutput{TV{[}TENSE=past, NUM=?n{]} -> 'saw' | 'liked'}}\end{alltt}
\caption{Example Feature-Based Grammar}
\vspace{1ex}
\hrule
\end{pylisting}

{\noindent} Notice that a syntactic category can have more than one feature; for example,
\docutilsrolecategory{v}{[}\docutilsrolefeature{tense}=\docutilsrolefval{pres}, \docutilsrolefeature{num}=\docutilsrolefval{pl}{]}.
In general, we can add as many features as we like.

Notice also that we have used feature variables in lexical entries as well
as grammatical productions. For example, \docutilsroleexample{the} has been assigned the
category \docutilsrolecategory{Det}{[}\docutilsrolefeature{num}=\docutilsrolemath{?n}{]}. Why is this?  Well,
you know that the definite article \docutilsroleexample{the} can combine with both
singular and plural nouns. One way of describing this would be to add
two lexical entries to the grammar, one each for the singular and
plural versions of \docutilsroleexample{the}. However, a more elegant solution is to
leave the \docutilsrolefeature{num} value \index{underspecified}\textbf{underspecified} and letting it agree
in number with whatever noun it combines with.

A final detail about \href{\#feat0cfg}{10.1} is the statement \texttt{{\%}{\linebreak[0]}start{\linebreak[0]} {\linebreak[0]}S}.
This a ''directive`` that tells the parser to take \docutilsrolecategory{s} as the
start symbol for the grammar.

In general, when we are trying to develop even a very small grammar,
it is convenient to put the productions in a file where they can be edited,
tested and revised.  We have saved \href{\#feat0cfg}{10.1} as a file named
\texttt{\pysrcstring{'{\linebreak[0]}feat0{\linebreak[0]}.{\linebreak[0]}fcfg{\linebreak[0]}'}} in the NLTK data distribution, and it
can be accessed using \texttt{nltk{\linebreak[0]}.{\linebreak[0]}data{\linebreak[0]}.{\linebreak[0]}load{\linebreak[0]}({\linebreak[0]})}.
% TODO: why two print statements here, print vs pprint? 
% TODO: first mention of earley in this chapter 

{\noindent} We can inspect the productions and the lexicon using the commands \texttt{\pysrckeyword{print}
g{\linebreak[0]}.{\linebreak[0]}earley{\_}grammar{\linebreak[0]}({\linebreak[0]})} and  \texttt{pprint{\linebreak[0]}({\linebreak[0]}g{\linebreak[0]}.{\linebreak[0]}earley{\_}lexicon{\linebreak[0]}({\linebreak[0]}){\linebreak[0]})}.

Next, we can tokenize a sentence and use the \texttt{nbest{\_}parse{\linebreak[0]}({\linebreak[0]})} function to
invoke the Earley chart parser.
\hypertarget{featurecharttrace}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tokens = \pysrcstring{'Kim likes children'}.split()
\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk.parse \pysrckeyword{import} load{\_}earley
\pysrcprompt{>{}>{}> }cp = load{\_}earley(\pysrcstring{'grammars/feat0.fcfg'}, trace=2)
\pysrcprompt{>{}>{}> }trees = cp.nbest{\_}parse(tokens)
\pysrcoutput{          |.K.l.c.|}
\pysrcoutput{Processing queue 0}
\pysrcoutput{Predictor |> . . .| {[}0:0{]} S{[}{]} -> * NP{[}NUM=?n{]} VP{[}NUM=?n{]} {\{}{\}}}
\pysrcoutput{Predictor |> . . .| {[}0:0{]} NP{[}NUM=?n{]} -> * N{[}NUM=?n{]} {\{}{\}}}
\pysrcoutput{Predictor |> . . .| {[}0:0{]} NP{[}NUM=?n{]} -> * PropN{[}NUM=?n{]} {\{}{\}}}
\pysrcoutput{Predictor |> . . .| {[}0:0{]} NP{[}NUM=?n{]} -> * Det{[}NUM=?n{]} N{[}NUM=?n{]} {\{}{\}}}
\pysrcoutput{Predictor |> . . .| {[}0:0{]} NP{[}NUM='pl'{]} -> * N{[}NUM='pl'{]} {\{}{\}}}
\pysrcoutput{Scanner   |{[}-{]} . .| {[}0:1{]} 'Kim'}
\pysrcoutput{Scanner   |{[}-{]} . .| {[}0:1{]} PropN{[}NUM='sg'{]} -> 'Kim' *}
\pysrcoutput{Processing queue 1}
\pysrcoutput{Completer |{[}-{]} . .| {[}0:1{]} NP{[}NUM='sg'{]} -> PropN{[}NUM='sg'{]} *}
\pysrcoutput{Completer |{[}-> . .| {[}0:1{]} S{[}{]} -> NP{[}NUM=?n{]} * VP{[}NUM=?n{]} {\{}?n: 'sg'{\}}}
\pysrcoutput{Predictor |. > . .| {[}1:1{]} VP{[}NUM=?n, TENSE=?t{]} -> * IV{[}NUM=?n, TENSE=?t{]} {\{}{\}}}
\pysrcoutput{Predictor |. > . .| {[}1:1{]} VP{[}NUM=?n, TENSE=?t{]} -> * TV{[}NUM=?n, TENSE=?t{]} NP{[}{]} {\{}{\}}}
\pysrcoutput{Scanner   |. {[}-{]} .| {[}1:2{]} 'likes'}
\pysrcoutput{Scanner   |. {[}-{]} .| {[}1:2{]} TV{[}NUM='sg', TENSE='pres'{]} -> 'likes' *}
\pysrcoutput{Processing queue 2}
\pysrcoutput{Completer |. {[}-> .| {[}1:2{]} VP{[}NUM=?n, TENSE=?t{]} -> TV{[}NUM=?n, TENSE=?t{]} * NP{[}{]} {\{}?n: 'sg', ?t: 'pres'{\}}}
\pysrcoutput{Predictor |. . > .| {[}2:2{]} NP{[}NUM=?n{]} -> * N{[}NUM=?n{]} {\{}{\}}}
\pysrcoutput{Predictor |. . > .| {[}2:2{]} NP{[}NUM=?n{]} -> * PropN{[}NUM=?n{]} {\{}{\}}}
\pysrcoutput{Predictor |. . > .| {[}2:2{]} NP{[}NUM=?n{]} -> * Det{[}NUM=?n{]} N{[}NUM=?n{]} {\{}{\}}}
\pysrcoutput{Predictor |. . > .| {[}2:2{]} NP{[}NUM='pl'{]} -> * N{[}NUM='pl'{]} {\{}{\}}}
\pysrcoutput{Scanner   |. . {[}-{]}| {[}2:3{]} 'children'}
\pysrcoutput{Scanner   |. . {[}-{]}| {[}2:3{]} N{[}NUM='pl'{]} -> 'children' *}
\pysrcoutput{Processing queue 3}
\pysrcoutput{Completer |. . {[}-{]}| {[}2:3{]} NP{[}NUM='pl'{]} -> N{[}NUM='pl'{]} *}
\pysrcoutput{Completer |. {[}-{}-{}-{]}| {[}1:3{]} VP{[}NUM='sg', TENSE='pres'{]} -> TV{[}NUM='sg', TENSE='pres'{]} NP{[}{]} *}
\pysrcoutput{Completer |{[}====={]}| {[}0:3{]} S{[}{]} -> NP{[}NUM='sg'{]} VP{[}NUM='sg'{]} *}
\pysrcoutput{Completer |{[}====={]}| {[}0:3{]} {[}INIT{]}{[}{]} -> S{[}{]} *}}\end{alltt}
\caption{Trace of Feature-Based Chart Parser}
\vspace{1ex}
\hrule
\end{pylisting}
% TODO: discussion of underspecified productions, compilation etc presumes specialized CL knowledge.
% Do we need to make this point?  Its just an implementation detail. 

{\noindent} Observe that the parser works directly with
the underspecified productions given by the grammar. That is, the
Predictor rule does not attempt to compile out all admissible feature
combinations before trying to expand the non-terminals on the left hand
side of a production. However, when the Scanner matches an input word
against a lexical production that has been predicted, the new edge will
typically contain fully specified features; e.g., the edge
{[}PropN{[}\docutilsrolefeature{num} = \docutilsrolefval{sg}{]} → 'Kim', (0, 1){]}. Recall from
\href{\#chap-parse}{Chapter 7} that the Fundamental (or Completer) Rule in
standard CFGs is used to combine an incomplete edge that's expecting a
nonterminal \emph{B} with a following, complete edge whose left hand side
matches \emph{B}. In our current setting, rather than checking for a
complete match, we test whether the expected category \emph{B} will
\index{unify}\textbf{unify} with the left hand side \emph{B'} of a following complete
edge. We will explain in more detail in \href{\#sec-feat-comp}{Section 10.3} how
unification works; for the moment, it is enough to know that as a
result of unification, any variable values of features in \emph{B} will be
instantiated by constant values in the corresponding feature structure
in \emph{B'}, and these instantiated values will be used in the new edge
added by the Completer. This instantiation can be seen, for example,
in the edge
{[}\docutilsrolecategory{np}{[}\docutilsrolefeature{num}=\docutilsrolefval{sg}{]} → PropN{[}\docutilsrolefeature{num}=\docutilsrolefval{sg}{]} •, (0, 1){]}
in \href{\#featurecharttrace}{10.2},
where the feature \docutilsrolefeature{num} has been assigned the value \docutilsrolefval{sg}.

Finally, we can inspect the resulting parse trees (in this case, a
single one).
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} tree \pysrckeyword{in} trees: \pysrckeyword{print} tree
\pysrcoutput{(S{[}{]}}
\pysrcoutput{  (NP{[}NUM='sg'{]} (PropN{[}NUM='sg'{]} Kim))}
\pysrcoutput{  (VP{[}NUM='sg', TENSE='pres'{]}}
\pysrcoutput{    (TV{[}NUM='sg', TENSE='pres'{]} likes)}
\pysrcoutput{    (NP{[}NUM='pl'{]} (N{[}NUM='pl'{]} children))))}}\end{alltt}
% TODO: delete the [INIT] and Start non terminals from the tree display! 


%___________________________________________________________________________

\subsection{Terminology}
\label{terminology}

So far, we have only seen feature values like \docutilsrolefval{sg} and
\docutilsrolefval{pl}. These simple values are usually called \index{atomic}\textbf{atomic}
— that is, they can't be decomposed into subparts. A special
case of atomic values are \index{boolean}\textbf{boolean} values, that is, values that
just specify whether a property is true or false of a category. For
example, we might want to distinguish \index{auxiliary}\textbf{auxiliary} verbs such as
\docutilsroleexample{can}, \docutilsroleexample{may}, \docutilsroleexample{will} and \docutilsroleexample{do} with the boolean feature
\docutilsrolefeature{aux}. Then our lexicon for verbs could include entries such as
\href{\#lex}{(90)}. (Note that we follow the convention that boolean
features are not written \docutilsrolefeature{f} +, \docutilsrolefeature{f} \docutilsrolemath{-} but simply
\docutilsrolefeature{+f}, \docutilsrolefeature{-f}, respectively.)
\hypertarget{lex}{}\begin{itemize}
\item[(90)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{V}{[}\docutilsrolefeature{tense}=\docutilsrolefval{pres},~\docutilsrolefeature{+aux}=\docutilsrolemath{+}{]}~→~'can'~\\
\docutilsrolecategory{V}{[}\docutilsrolefeature{tense}=\docutilsrolefval{pres},~\docutilsrolefeature{+aux}=\docutilsrolemath{+}{]}~→~'may'~\\
~\\
\docutilsrolecategory{V}{[}\docutilsrolefeature{tense}=\docutilsrolefval{pres},~\docutilsrolefeature{-aux}~\docutilsrolemath{-}{]}~→~'walks'~\\
\docutilsrolecategory{V}{[}\docutilsrolefeature{tense}=\docutilsrolefval{pres},~\docutilsrolefeature{-aux}~\docutilsrolemath{-}{]}~→~'likes'
}\end{quote}
\end{itemize}

We have spoken informally of attaching ''feature annotations`` to
syntactic categories. A more general
approach is to treat the whole category — that is, the
non-terminal symbol plus the annotation — as a bundle of
features. Consider, for example, the object we have written as \href{\#ncat0}{(91)}.
\hypertarget{ncat0}{}\begin{itemize}
\item[(91)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{n}{[}\docutilsrolefeature{num}=\docutilsrolefval{sg}{]}
}\end{quote}
\end{itemize}

{\noindent} The syntactic category \docutilsrolecategory{n}, as we have seen before, provides part
of speech information. This information can itself be captured as a
feature value pair, using  \docutilsrolefeature{pos} to represent ''part of speech``:
\hypertarget{ncat1}{}\begin{itemize}
\item[(92)] \begin{quote}{\ttfamily \raggedright \noindent
\small
{[}\docutilsrolefeature{pos}=\docutilsrolefval{N},~\docutilsrolefeature{num}=\docutilsrolefval{sg}{]}
}\end{quote}
\end{itemize}

{\noindent} In fact, we  regard \href{\#ncat1}{(92)} as our ''official`` representation of a
feature-based linguistic category, and \href{\#ncat0}{(91)} as a convenient abbreviation.
A bundle of feature-value pairs is called a \index{feature structure}\textbf{feature structure}
or an \index{attribute value matrix}\textbf{attribute value matrix} (AVM). A feature structure that
contains a specification for the feature \docutilsrolefeature{pos} is a \index{linguistic
category}\textbf{linguistic
category}.

In addition to atomic-valued features, we allow features whose values
are themselves feature structures. For example, we might want to group
together agreement features (e.g., person, number and gender) as a
distinguished part of a category, as shown in \href{\#agr0}{(93)}.
\hypertarget{agr0}{}\begin{itemize}
\item[(93)] 
\begin{avm}
\[
 pos & N \\
 agr & \[
     per & 3 \\
     num & pl \\
     gnd & fem\]
\]
\end{avm}

\end{itemize}

{\noindent} In this case, we say that the feature \docutilsrolefeature{agr} has a \index{complex}\textbf{complex} value.

There is no particular significance to the \emph{order} of features in a
feature structure. So \href{\#agr0}{(93)} is equivalent to \href{\#agr0}{(93)}.
\hypertarget{agr1}{}\begin{itemize}
\item[(94)] 
\begin{avm}
\[
 agr & \[
     num & pl \\
     per & 3 \\
     gnd & fem\]
 \\
 pos & N\]
\end{avm}

\end{itemize}

Once we have the possibility of using features like \docutilsrolefeature{agr}, we
can refactor a grammar like \href{\#feat0cfg}{10.1} so that agreement features are
bundled together. A tiny grammar illustrating this point is shown in \href{\#agr2}{(95)}.
\hypertarget{agr2}{}\begin{itemize}
\item[(95)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{s}~→~\docutilsrolecategory{np}{[}\docutilsrolefeature{agr}=\docutilsrolefval{?n}{]}~\docutilsrolecategory{vp}{[}\docutilsrolefeature{agr}=\docutilsrolefval{?n}{]}~\\
\docutilsrolecategory{np}{[}\docutilsrolefeature{agr}=\docutilsrolefval{?n}{]}~→~\docutilsrolecategory{PropN}{[}\docutilsrolefeature{agr}=\docutilsrolefval{?n}{]}~\\
\docutilsrolecategory{vp}{[}\docutilsrolefeature{tense}=\docutilsrolefval{?t},~\docutilsrolefeature{agr}=\docutilsrolefval{?n}{]}~→~\docutilsrolecategory{Cop}{[}\docutilsrolefeature{tense}=\docutilsrolefval{?t},~\docutilsrolefeature{agr}=\docutilsrolefval{?n}{]}~Adj~\\
\docutilsrolecategory{Cop}{[}\docutilsrolefeature{tense}=\docutilsrolefval{pres},~~\docutilsrolefeature{agr}={[}\docutilsrolefeature{num}=\docutilsrolefval{sg},~\docutilsrolefeature{per}=\docutilsrolefval{3}{]}{]}~→~'is'~\\
\docutilsrolecategory{PropN}{[}\docutilsrolefeature{agr}={[}\docutilsrolefeature{num}=\docutilsrolefval{sg},~\docutilsrolefeature{per}=\docutilsrolefval{3}{]}{]}~→~'Kim'~\\
\docutilsrolecategory{Adj}~→~'happy'
}\end{quote}
\end{itemize}


%___________________________________________________________________________

\subsection{Exercises}
\label{id97}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ What constraints are required to correctly parse strings like \docutilsroleexample{I am
happy} and \docutilsroleexample{she is happy} but not \docutilsroleexample{*you is happy} or
\docutilsroleexample{*they am happy}? Implement two solutions for the present tense
paradigm of the verb \docutilsroleexample{be} in English, first taking Grammar
\href{\#agcfg1}{(81)} as your starting point, and then taking Grammar \href{\#agr2}{(95)}
as the starting point.

\item {} 
☼ Develop a variant of grammar \href{\#feat0cfg}{10.1} that uses a
feature \docutilsrolefeature{count} to make the distinctions shown below:
\begin{itemize}
\item[(96)] \begin{itemize}
\item[a.] 
The boy sings.
\end{itemize}
\begin{itemize}
\item[b.] 
*Boy sings.
\end{itemize}
\end{itemize}
\begin{itemize}
\item[(97)] \begin{itemize}
\item[a.] 
The boys sing.
\end{itemize}
\begin{itemize}
\item[b.] 
Boys sing.
\end{itemize}
\end{itemize}
\begin{itemize}
\item[(98)] \begin{itemize}
\item[a.] 
The boys sing.
\end{itemize}
\begin{itemize}
\item[b.] 
Boys sing.
\end{itemize}
\end{itemize}
\begin{itemize}
\item[(99)] \begin{itemize}
\item[a.] 
The water is precious.
\end{itemize}
\begin{itemize}
\item[b.] 
Water is precious.
\end{itemize}
\end{itemize}

\item {} 
◑ Develop a feature-based grammar that will correctly describe the following
Spanish noun phrases:
\begin{itemize}
\item[(100)] 
\begin{tabular}[c]{p{0.203\locallinewidth}p{0.086\locallinewidth}p{0.121\locallinewidth}}

un
 & 
cuadro
 & 
hermos-o
 \\

INDEF.SG.MASC
 & 
picture
 & 
beautiful-SG.MASC
 \\
\multicolumn{3}{l}{
'a beautiful picture'
} \\
\end{tabular}
\end{itemize}
\begin{itemize}
\item[(101)] 
\begin{tabular}[c]{p{0.203\locallinewidth}p{0.086\locallinewidth}p{0.121\locallinewidth}}

un-os
 & 
cuadro-s
 & 
hermos-os
 \\

INDEF-PL.MASC
 & 
picture-PL
 & 
beautiful-PL.MASC
 \\
\multicolumn{3}{l}{
'beautiful pictures'
} \\
\end{tabular}
\end{itemize}
\begin{itemize}
\item[(102)] 
\begin{tabular}[c]{p{0.203\locallinewidth}p{0.086\locallinewidth}p{0.110\locallinewidth}}

un-a
 & 
cortina
 & 
hermos-a
 \\

INDEF-SG.FEM
 & 
curtain
 & 
beautiful-SG.FEM
 \\
\multicolumn{3}{l}{
'a beautiful curtain'
} \\
\end{tabular}
\end{itemize}
\begin{itemize}
\item[(103)] 
\begin{tabular}[c]{p{0.203\locallinewidth}p{0.086\locallinewidth}p{0.110\locallinewidth}}

un-as
 & 
cortina-s
 & 
hermos-as
 \\

INDEF-PL.FEM
 & 
curtain
 & 
beautiful-PL.FEM
 \\
\multicolumn{3}{l}{
'beautiful curtains'
} \\
\end{tabular}
\end{itemize}

\item {} 
◑ Develop a wrapper for the \texttt{earley{\_}parser} so that a trace
is only printed if the input string fails to parse.

\end{list}
\hypertarget{sec-feat-comp}{}

%___________________________________________________________________________

\section{Computing with Feature Structures}
\label{computing-with-feature-structures}
\label{sec-feat-comp}

In this section, we will show how feature structures can be
constructed and manipulated in Python. We will also discuss the
fundamental operation of unification, which allows us to combine the
information contained in two different feature structures.


%___________________________________________________________________________

\subsection{Feature Structures in Python}
\label{feature-structures-in-python}

Feature structures are declared with the
\texttt{FeatStruct{\linebreak[0]}({\linebreak[0]})} constructor. Atomic feature values can be strings or
integers.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fs1 = nltk.FeatStruct(TENSE=\pysrcstring{'past'}, NUM=\pysrcstring{'sg'})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} fs1
\pysrcoutput{{[} NUM   = 'sg'   {]}}
\pysrcoutput{{[} TENSE = 'past' {]}}}\end{alltt}

{\noindent} A feature structure is actually just a kind of dictionary,
and so we access its values by indexing in the usual way.
We can use our familiar syntax to \emph{assign} values to features:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fs1 = nltk.FeatStruct(PER=3, NUM=\pysrcstring{'pl'}, GND=\pysrcstring{'fem'})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} fs1{[}\pysrcstring{'GND'}{]}
\pysrcoutput{fem}
\pysrcoutput{}\pysrcprompt{>{}>{}> }fs1{[}\pysrcstring{'CASE'}{]} = \pysrcstring{'acc'}}\end{alltt}

{\noindent} We can also define feature structures that have complex values, as
discussed earlier.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fs2 = nltk.FeatStruct(POS=\pysrcstring{'N'}, AGR=fs1)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} fs2
\pysrcoutput{{[}       {[} CASE = 'acc' {]} {]}}
\pysrcoutput{{[} AGR = {[} GND  = 'fem' {]} {]}}
\pysrcoutput{{[}       {[} NUM  = 'pl'  {]} {]}}
\pysrcoutput{{[}       {[} PER  = 3     {]} {]}}
\pysrcoutput{{[}                        {]}}
\pysrcoutput{{[} POS = 'N'              {]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{print} fs2{[}\pysrcstring{'AGR'}{]}
\pysrcoutput{{[} CASE = 'acc' {]}}
\pysrcoutput{{[} GND  = 'fem' {]}}
\pysrcoutput{{[} NUM  = 'pl'  {]}}
\pysrcoutput{{[} PER  = 3     {]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{print} fs2{[}\pysrcstring{'AGR'}{]}{[}\pysrcstring{'PER'}{]}
\pysrcoutput{3}}\end{alltt}

An alternative method of specifying feature structures is to
use a bracketed string consisting of feature-value pairs in the format
\texttt{feature{\linebreak[0]}={\linebreak[0]}value}, where values may themselves be feature structures:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.FeatStruct(\pysrcstring{"{[}POS='N', AGR={[}PER=3, NUM='pl', GND='fem'{]}{]}"})
\pysrcoutput{{[}AGR={[}GND='fem', NUM='pl', PER=3{]}, POS='N'{]}}}\end{alltt}


%___________________________________________________________________________

\subsection{Feature Structures as Graphs}
\label{feature-structures-as-graphs}

Feature structures are not inherently tied to linguistic objects; they are
general purpose structures for representing knowledge. For example, we
could encode information about a person in a feature structure:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }person01 = nltk.FeatStruct(name=\pysrcstring{'Lee'}, telno=\pysrcstring{'01 27 86 42 96'}, age=33)}\end{alltt}
\hypertarget{person01}{}\begin{itemize}
\item[(104)] 
\begin{avm}
\[
 name & `Lee' \\
 telno & 01 27 86 42 96 \\
 age & 33\]
\end{avm}

\end{itemize}

It is sometimes helpful to view feature structures as graphs; more
specifically, \index{directed acyclic graphs}\textbf{directed acyclic graphs} (DAGs). \href{\#dag01}{(105)} is equivalent to
the AVM \href{\#person01}{(104)}.
\hypertarget{dag01}{}\begin{itemize}
\item[(105)] \scalebox{0.205714}{\includegraphics{../images/dag01.png}}\end{itemize}

{\noindent} The feature names appear as labels on the directed arcs, and feature
values appear as labels on the nodes that are pointed to by the arcs.

Just as before, feature values can be complex:
\hypertarget{dag02}{}\begin{itemize}
\item[(106)] \scalebox{0.205714}{\includegraphics{../images/dag02.png}}\end{itemize}

{\noindent} When we look at such graphs, it is natural to think in terms of
paths through the graph. A \index{feature path}\textbf{feature path} is a sequence of arcs
that can be followed from the root node. We will represent paths as
tuples. Thus, \texttt{(\pysrcstring{'{\linebreak[0]}address{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}street{\linebreak[0]}'})} is a feature path whose value
in \href{\#dag02}{(106)} is the string ''rue Pascal``.

Now let's consider a situation where Lee has a spouse named ''Kim``, and
Kim's address is the same as Lee's.
We might represent this as \href{\#dag04}{(107)}.
\hypertarget{dag04}{}\begin{itemize}
\item[(107)] \scalebox{0.205714}{\includegraphics{../images/dag04.png}}\end{itemize}

{\noindent} However, rather than repeating the address
information in the feature structure, we can ''share`` the same
sub-graph between different arcs:
\hypertarget{dag03}{}\begin{itemize}
\item[(108)] \scalebox{0.205714}{\includegraphics{../images/dag03.png}}\end{itemize}

{\noindent} In other words, the value of the path \texttt{(\pysrcstring{'{\linebreak[0]}ADDRESS{\linebreak[0]}'})} in \href{\#dag03}{(108)} is
identical to the value of the path \texttt{(\pysrcstring{'{\linebreak[0]}SPOUSE{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}ADDRESS{\linebreak[0]}'})}.  DAGs
such as \href{\#dag03}{(108)} are said to involve \index{structure sharing}\textbf{structure sharing} or
\index{reentrancy}\textbf{reentrancy}. When two paths have the same value, they are said to
be \index{equivalent}\textbf{equivalent}.

There are a number of notations for representing reentrancy in
matrix-style representations of feature structures. We adopt
the following convention: the first occurrence of a shared feature structure
is prefixed with an integer in parentheses, such as \texttt{({\linebreak[0]}1{\linebreak[0]})}, and any
subsequent reference to that structure uses the notation
\texttt{-{\linebreak[0]}>{\linebreak[0]}({\linebreak[0]}1{\linebreak[0]})}, as shown below.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fs = nltk.FeatStruct(\pysrcstring{"{}"{}"{[}NAME='Lee', ADDRESS=(1){[}NUMBER=74, STREET='rue Pascal'{]},}
\pysrcmore{... }\pysrcstring{                        SPOUSE={[}NAME='Kim', ADDRESS->(1){]}{]}"{}"{}"})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} fs
\pysrcoutput{{[} ADDRESS = (1) {[} NUMBER = 74           {]} {]}}
\pysrcoutput{{[}               {[} STREET = 'rue Pascal' {]} {]}}
\pysrcoutput{{[}                                         {]}}
\pysrcoutput{{[} NAME    = 'Lee'                         {]}}
\pysrcoutput{{[}                                         {]}}
\pysrcoutput{{[} SPOUSE  = {[} ADDRESS -> (1)  {]}           {]}}
\pysrcoutput{{[}           {[} NAME    = 'Kim' {]}           {]}}}\end{alltt}

{\noindent} This is similar to more conventional displays of AVMs, as shown in
\href{\#reentrant01}{(109)}.
\hypertarget{reentrant01}{}\begin{itemize}
\item[(109)] 
\begin{avm}
\[
 address & \@1 \[
     number & 74 \\
     street & 'rue Pascal'\]
 \\
 name & 'Lee' \\
 spouse & \[
     address & \@{1} \\
     name & 'Kim'\]
\]
\end{avm}

\end{itemize}

{\noindent} The bracketed integer is sometimes called a \index{tag}\textbf{tag} or a
\index{coindex}\textbf{coindex}. The choice of integer is not significant.
There can be any number of tags within a single feature structure.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fs1 = nltk.FeatStruct(\pysrcstring{"{[}A='a', B=(1){[}C='c'{]}, D->(1), E->(1){]}"})}\end{alltt}
\hypertarget{reentrant02}{}\begin{itemize}
\item[(110)] 
\begin{avm}
\[
 A & 'a' \\
 B & \@1 \[
     C & 'c'\]
 \\
 D & \@{1} \\
 E & \@{1}\]
\end{avm}

\end{itemize}
% TODO following AVM doesn't currently parse 
% |nopar| We can also share empty structures:
% 
%     >>> fs2 = nltk.FeatStruct("[A=(1)[], B=(2)[], C->(1), D->(2)]")
% 
% .. _reentrant03:
% .. ex::
%       .. avm::
% 
%          [ A = (1) [ ] ]
%          [ B = (2) [ ] ]
%          [ C -> (1)    ]
%          [ D -> (2)    ] 


%___________________________________________________________________________

\subsection{Subsumption and Unification}
\label{subsumption-and-unification}

It is standard to think of feature structures as providing \index{partial
information}\textbf{partial
information} about some object, in the sense that we can order
feature structures according to how general they are. For example,
\href{\#fs01}{(111a)} is more general (less specific) than \href{\#fs02}{(111b)}, which in turn is more general than \href{\#fs03}{(111c)}.
\begin{itemize}
\item[(111)] \hypertarget{fs01}{}\begin{itemize}
\item[a.] 
\begin{avm}
\[
 number & 74\]
\end{avm}

\end{itemize}
\hypertarget{fs02}{}\begin{itemize}
\item[b.] 
\begin{avm}
\[
 number & 74 \\
 street & 'rue Pascal'\]
\end{avm}

\end{itemize}
\hypertarget{fs03}{}\begin{itemize}
\item[c.] 
\begin{avm}
\[
 number & 74 \\
 street & 'rue Pascal' \\
 city & 'Paris'\]
\end{avm}

\end{itemize}
\end{itemize}

{\noindent} This ordering is called \index{subsumption}\textbf{subsumption}; a more general feature
structure \index{subsumes}\textbf{subsumes} a less general one. If \docutilsrolemath{FS}$_{0}$ subsumes \docutilsrolemath{FS}$_{1}$ (formally, we write
\docutilsrolemath{FS}$_{0}$ ⊑ \docutilsrolemath{FS}$_{1}$), then \docutilsrolemath{FS}$_{1}$ must have all the
paths and path equivalences of \docutilsrolemath{FS}$_{0}$, and may
have additional paths and equivalences as well. Thus, \href{\#dag04}{(107)} subsumes
\href{\#dag03}{(108)}, since the latter has additional path equivalences.. It should
be obvious that subsumption only provides a partial ordering on
feature structures, since some feature structures are
incommensurable. For example, \href{\#fs04}{(112)} neither subsumes nor is subsumed
by \href{\#fs01}{(111a)}.
\hypertarget{fs04}{}\begin{itemize}
\item[(112)] 
\begin{avm}
\[
 telno & 01 27 86 42 96\]
\end{avm}

\end{itemize}

So we have seen that some feature structures are more specific than
others. How do we go about specializing a given feature structure?
For example, we might decide that addresses should
consist of not just a street number and a street name, but also a
city. That is, we might want to \emph{merge}  graph \href{\#dag042}{(113b)} with \href{\#dag041}{(113a)} to
yield \href{\#dag043}{(113c)}.
\begin{itemize}
\item[(113)] \hypertarget{dag041}{}\begin{itemize}
\item[a.] \scalebox{0.205714}{\includegraphics{../images/dag04-1.png}}\end{itemize}
\hypertarget{dag042}{}\begin{itemize}
\item[b.] \scalebox{0.205714}{\includegraphics{../images/dag04-2.png}}\end{itemize}
\hypertarget{dag043}{}\begin{itemize}
\item[c.] \scalebox{0.205714}{\includegraphics{../images/dag04-3.png}}\end{itemize}
\end{itemize}

{\noindent} Merging information from two feature structures is called
\index{unification}\textbf{unification} and is supported by the \texttt{unify{\linebreak[0]}({\linebreak[0]})} method.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fs1 = nltk.FeatStruct(NUMBER=74, STREET=\pysrcstring{'rue Pascal'})
\pysrcprompt{>{}>{}> }fs2 = nltk.FeatStruct(CITY=\pysrcstring{'Paris'})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} fs1.unify(fs2)
\pysrcoutput{{[} CITY   = 'Paris'      {]}}
\pysrcoutput{{[} NUMBER = 74           {]}}
\pysrcoutput{{[} STREET = 'rue Pascal' {]}}}\end{alltt}

Unification is formally defined as a binary operation: \docutilsrolemath{FS}$_{0}$ $\sqcap$ \docutilsrolemath{FS}$_{1}$. Unification is symmetric, so
\begin{itemize}
\item[(114)] 
\docutilsrolemath{FS}$_{0}$ $\sqcap$ \docutilsrolemath{FS}$_{1}$ = \docutilsrolemath{FS}$_{1}$ $\sqcap$
\docutilsrolemath{FS}$_{0}$.
\end{itemize}

{\noindent} The same is true in Python:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} fs2.unify(fs1)
\pysrcoutput{{[} CITY   = 'Paris'      {]}}
\pysrcoutput{{[} NUMBER = 74           {]}}
\pysrcoutput{{[} STREET = 'rue Pascal' {]}}}\end{alltt}
% TODO: also mention commutativity 
% but >>> fs1.unify(fs2) is fs2.unify(fs1)
%     False
% only works with repr() 

If we unify two feature structures which stand in the subsumption
relationship, then the result of unification is the most specific of
the two:
\begin{itemize}
\item[(115)] 
If \docutilsrolemath{FS}$_{0}$ ⊑ \docutilsrolemath{FS}$_{1}$,  then \docutilsrolemath{FS}$_{0}$
$\sqcap$ \docutilsrolemath{FS}$_{1}$ = \docutilsrolemath{FS}$_{1}$
\end{itemize}

{\noindent} For example, the result of unifying \href{\#fs02}{(111b)} with \href{\#fs03}{(111c)} is \href{\#fs03}{(111c)}.

Unification between \docutilsrolemath{FS}$_{0}$ and \docutilsrolemath{FS}$_{1}$ will fail if the two feature structures share a path π,
but the value of π in \docutilsrolemath{FS}$_{0}$ is a distinct
atom from the value of π in \docutilsrolemath{FS}$_{1}$.
This is implemented by setting the result of unification to be \texttt{None}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fs0 = nltk.FeatStruct(A=\pysrcstring{'a'})
\pysrcprompt{>{}>{}> }fs1 = nltk.FeatStruct(A=\pysrcstring{'b'})
\pysrcprompt{>{}>{}> }fs2 = fs0.unify(fs1)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} fs2
\pysrcoutput{None}}\end{alltt}

Now, if we look at how unification interacts with structure-sharing,
things become really interesting. First, let's define \href{\#dag04}{(107)} in Python:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fs0 = nltk.FeatStruct(\pysrcstring{"{}"{}"{[}NAME=Lee,}
\pysrcmore{... }\pysrcstring{                          ADDRESS={[}NUMBER=74,}
\pysrcmore{... }\pysrcstring{                                   STREET='rue Pascal'{]},}
\pysrcmore{... }\pysrcstring{                          SPOUSE= {[}NAME=Kim,}
\pysrcmore{... }\pysrcstring{                                   ADDRESS={[}NUMBER=74,}
\pysrcmore{... }\pysrcstring{                                            STREET='rue Pascal'{]}{]}{]}"{}"{}"})}\end{alltt}
\hypertarget{unification01}{}\begin{itemize}
\item[(116)] 
\begin{avm}
\[
 address & \[
     number & 74 \\
     street & `rue Pascal'\]
 \\
 name & `Lee' \\
 spouse & \[
     address & \[
         number & 74 \\
         street & `rue Pascal'\]
 \\
     name & `Kim'\]
\]
\end{avm}

\end{itemize}

{\noindent} What happens when we augment Kim's address with a specification
for \docutilsrolefeature{city}? (Notice that \texttt{fs1} includes the whole path from the root of
the feature structure down to \docutilsrolefeature{city}.)
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fs1 = nltk.FeatStruct(\pysrcstring{"{[}SPOUSE = {[}ADDRESS = {[}CITY = Paris{]}{]}{]}"})}\end{alltt}

\href{\#unification02}{(117)} shows the result of unifying \texttt{fs0} with \texttt{fs1}:
\hypertarget{unification02}{}\begin{itemize}
\item[(117)] 
\begin{avm}
\[
 address & \[
     number & 74 \\
     street & `rue Pascal'\]
 \\
 name & `Lee' \\
 spouse & \[
     address & \[
         city & `Paris' \\
         number & 74 \\
         street & `rue Pascal'\]
 \\
     name & `Kim'\]
\]
\end{avm}

\end{itemize}

{\noindent} By contrast, the result is very different if \texttt{fs1} is unified with
the structure-sharing version \texttt{fs2} (also shown as \href{\#dag03}{(108)}):
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fs2 = nltk.FeatStruct(\pysrcstring{"{}"{}"{[}NAME=Lee, ADDRESS=(1){[}NUMBER=74, STREET='rue Pascal'{]},}
\pysrcmore{... }\pysrcstring{                          SPOUSE={[}NAME=Kim, ADDRESS->(1){]}{]}"{}"{}"})}\end{alltt}
\hypertarget{unification03}{}\begin{itemize}
\item[(118)] 
\begin{avm}
\[
 address & \@1 \[
     city & `Paris' \\
     number & 74 \\
     street & `rue Pascal'\]
 \\
 name & `Lee' \\
 spouse & \[
     address & \@{1} \\
     name & `Kim'\]
\]
\end{avm}

\end{itemize}

{\noindent} Rather than just updating what was in effect Kim's ''copy`` of Lee's address,
we have now updated \docutilsroleemphasis{both} their addresses at the same time. More
generally, if a unification involves specializing the value of some
path π, then that unification simultaneously specializes the value
of \docutilsroleemphasis{any path that is equivalent to} π.

As we have already seen, structure sharing can also be stated
using variables such as \texttt{?{\linebreak[0]}x}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fs1 = nltk.FeatStruct(\pysrcstring{"{[}ADDRESS1={[}NUMBER=74, STREET='rue Pascal'{]}{]}"})
\pysrcprompt{>{}>{}> }fs2 = nltk.FeatStruct(\pysrcstring{"{[}ADDRESS1=?x, ADDRESS2=?x{]}"})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} fs2
\pysrcoutput{{[} ADDRESS1 = ?x {]}}
\pysrcoutput{{[} ADDRESS2 = ?x {]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{print} fs2.unify(fs1)
\pysrcoutput{{[} ADDRESS1 = (1) {[} NUMBER = 74           {]} {]}}
\pysrcoutput{{[}                {[} STREET = 'rue Pascal' {]} {]}}
\pysrcoutput{{[}                                          {]}}
\pysrcoutput{{[} ADDRESS2 -> (1)                          {]}}}\end{alltt}


%___________________________________________________________________________

\subsection{Exercises}
\label{id98}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Write a function \titlereference{subsumes()} which holds of two feature
structures \texttt{fs1} and \texttt{fs2} just in case \texttt{fs1} subsumes \texttt{fs2}.

\item {} 
◑ Consider the feature structures shown in \href{\#featstructures}{Listing 10.3}.
\begin{figure}[b]\hypertarget{xx}[XX]
NOTE: This example is somewhat broken -{}- nltk doesn't support
reentrance for base feature values.  (See email {\textasciitilde}7/23/08 to the
nltk-users mailing list for details.)
\end{figure}
\hypertarget{featstructures}{}
\begin{pylisting}
\begin{alltt}\small\textbf{fs1 = nltk.FeatStruct(\pysrcstring{"{[}A = (1)b, B= {[}C ->(1){]}{]}"})
fs2 = nltk.FeatStruct(\pysrcstring{"{[}B = {[}D = d{]}{]}"})
fs3 = nltk.FeatStruct(\pysrcstring{"{[}B = {[}C = d{]}{]}"})
fs4 = nltk.FeatStruct(\pysrcstring{"{[}A = (1){[}B = b{]}, C->(1){]}"})
fs5 = nltk.FeatStruct(\pysrcstring{"{[}A = {[}D = (1)e{]}, C = {[}E -> (1){]} {]}"})
fs6 = nltk.FeatStruct(\pysrcstring{"{[}A = {[}D = (1)e{]}, C = {[}B -> (1){]} {]}"})
fs7 = nltk.FeatStruct(\pysrcstring{"{[}A = {[}D = (1)e, F = (2){[}{]}{]}, C = {[}B -> (1), E -> (2){]} {]}"})
fs8 = nltk.FeatStruct(\pysrcstring{"{[}A = {[}B = b{]}, C = {[}E = {[}G = e{]}{]}{]}"})
fs9 = nltk.FeatStruct(\pysrcstring{"{[}A = (1){[}B = b{]}, C -> (1){]}"})}\end{alltt}
\caption{}
\vspace{1ex}
\hrule
\end{pylisting}

Work out on paper what the result is of the following
unifications. (Hint: you might find it useful to draw the graph structures.)
\setcounter{listcnt1}{0}
\begin{list}{\arabic{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
\texttt{fs1} and \texttt{fs2}

\item {} 
\texttt{fs1} and \texttt{fs3}

\item {} 
\texttt{fs4} and \texttt{fs5}

\item {} 
\texttt{fs5} and \texttt{fs6}

\item {} 
\texttt{fs7} and \texttt{fs8}

\item {} 
\texttt{fs7} and \texttt{fs9}

\end{list}

Check your answers using Python.

\item {} 
◑ List two feature structures that subsume {[}A=?x, B=?x{]}.

\item {} 
◑ Ignoring structure sharing, give an informal algorithm for unifying
two feature structures.

\end{list}


%___________________________________________________________________________

\section{Extending a Feature-Based Grammar}
\label{extending-a-feature-based-grammar}
% TODO: I think this section needs an opening 
\hypertarget{sec-subcat}{}

%___________________________________________________________________________

\subsection{Subcategorization}
\label{subcategorization}
\label{sec-subcat}

In \href{\#chap-parse}{Chapter 7}, we proposed to augment our
category labels to represent different kinds of verb.
We introduced labels such as \docutilsrolecategory{iv} and \docutilsrolecategory{tv} for intransitive
and transitive verbs respectively.  This allowed us to write productions
like the following:
\hypertarget{subcatcfg0}{}\begin{itemize}
\item[(119)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{vp}~→~\docutilsrolecategory{iv}~\\
\docutilsrolecategory{vp}~→~\docutilsrolecategory{tv~np}
}\end{quote}
\end{itemize}

{\noindent} Although we know that \docutilsrolecategory{iv} and \docutilsrolecategory{tv} are two
kinds of \docutilsrolecategory{v}, from a formal point of view
\docutilsrolecategory{iv} has no closer relationship with \docutilsrolecategory{tv} than it does
with \docutilsrolecategory{np}. As it stands, \docutilsrolecategory{iv} and \docutilsrolecategory{tv} are just atomic
nonterminal symbols from a CFG.  This approach doesn't allow us
to say anything about the class of verbs in general.
For example, we cannot say something like ''All lexical
items of category \docutilsrolecategory{v} can be marked for tense``, since \docutilsroleexample{bark},
say, is an item of category \docutilsrolecategory{iv}, not \docutilsrolecategory{v}.
A simple solution, originally developed for a grammar framework
called Generalized Phrase Structure Grammar (GPSG), stipulates that lexical
categories may bear a \docutilsrolefeature{subcat} feature whose values are integers.
This is illustrated in a modified portion of \href{\#feat0cfg}{10.1}, shown in \href{\#subcatgpsg}{(120)}.
\hypertarget{subcatgpsg}{}\begin{itemize}
\item[(120)] \begin{quote}{\ttfamily \raggedright \noindent
\small
VP{[}TENSE=?t,~NUM=?n{]}~->~V{[}SUBCAT=0,~TENSE=?t,~NUM=?n{]}~\\
VP{[}TENSE=?t,~NUM=?n{]}~->~V{[}SUBCAT=1,~TENSE=?t,~NUM=?n{]}~NP~\\
VP{[}TENSE=?t,~NUM=?n{]}~->~V{[}SUBCAT=2,~TENSE=?t,~NUM=?n{]}~Sbar~\\
~\\
V{[}SUBCAT=0,~TENSE=pres,~NUM=sg{]}~->~'disappears'~|~'walks'~\\
V{[}SUBCAT=1,~TENSE=pres,~NUM=sg{]}~->~'sees'~|~'likes'~\\
V{[}SUBCAT=2,~TENSE=pres,~NUM=sg{]}~->~'says'~|~'claims'~\\
~\\
V{[}SUBCAT=0,~TENSE=pres,~NUM=pl{]}~->~'disappear'~|~'walk'~\\
V{[}SUBCAT=1,~TENSE=pres,~NUM=pl{]}~->~'see'~|~'like'~\\
V{[}SUBCAT=2,~TENSE=pres,~NUM=pl{]}~->~'say'~|~'claim'~\\
~\\
V{[}SUBCAT=0,~TENSE=past,~NUM=?n{]}~->~'disappeared'~|~'walked'~\\
V{[}SUBCAT=1,~TENSE=past,~NUM=?n{]}~->~'saw'~|~'liked'~\\
V{[}SUBCAT=2,~TENSE=past,~NUM=?n{]}~->~'said'~|~'claimed'
}\end{quote}
\end{itemize}

{\noindent} When we see a lexical category like \docutilsrolecategory{v}{[}\docutilsrolefeature{subcat}
\docutilsrolefval{1}{]}, we can interpret the \docutilsrolefeature{subcat} specification as a
pointer to the production in which \docutilsrolecategory{v}{[}\docutilsrolefeature{subcat} \docutilsrolefval{1}{]}
is introduced as the head daughter in a \docutilsrolecategory{vp} production.
By convention, there is a one-to-one correspondence between
\docutilsrolefeature{subcat} values and the productions that introduce lexical heads.
It's worth noting that the choice of integer which acts as a value for
\docutilsrolefeature{subcat} is completely arbitrary — we could equally well
have chosen 3999, 113 and 57 as our two values in \href{\#subcatgpsg}{(120)}.  On this
approach, \docutilsrolefeature{subcat} can \emph{only} appear on lexical categories; it
makes no sense, for example, to specify a \docutilsrolefeature{subcat} value on
\docutilsrolecategory{vp}.

In our third class of verbs above, we have specified a category
\docutilsrolecategory{s-bar}. This is a label for subordinate clauses such as the
complement of \docutilsroleexample{claim} in the example \docutilsroleexample{You claim that you like
children}. We require two further productions to analyze such sentences:
\hypertarget{sbar}{}\begin{itemize}
\item[(121)] \begin{quote}{\ttfamily \raggedright \noindent
\small
S-BAR~->~Comp~S~\\
Comp~->~'that'
}\end{quote}
\end{itemize}

{\noindent} The resulting structure is the following.
\hypertarget{sbartree}{}\begin{itemize}
\item[(122)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-35.pdf}}}\end{itemize}

An alternative treatment of subcategorization, due originally to a framework
known as categorial grammar, is represented in feature-based frameworks such as PATR
and Head-driven Phrase Structure Grammar. Rather than using
\docutilsrolefeature{subcat} values as a way of indexing productions, the \docutilsrolefeature{subcat}
value directly encodes the valency of a head (the list of
arguments that it can combine with). For example, a verb like
\docutilsroleexample{put} that takes  \docutilsrolecategory{np} and \docutilsrolecategory{pp} complements (\docutilsroleexample{put the
book on the table}) might be represented as \href{\#subcathpsg0}{(123)}:
% TODO: angle brackets don't appear 
\hypertarget{subcathpsg0}{}\begin{itemize}
\item[(123)] 
\docutilsrolecategory{v}{[}\docutilsrolefeature{subcat} 〈\docutilsrolecategory{np}, \docutilsrolecategory{np}, \docutilsrolecategory{pp}〉 {]}
\end{itemize}

{\noindent} This says that the verb can combine with three  arguments. The
leftmost element in the list is the subject \docutilsrolecategory{np}, while everything
else — an \docutilsrolecategory{np} followed by a \docutilsrolecategory{pp} in this case — comprises the
subcategorized-for complements. When a verb like \docutilsroleexample{put} is combined
with appropriate complements, the requirements which are specified in
the  \docutilsrolefeature{subcat} are discharged, and only a subject \docutilsrolecategory{np} is
needed. This category, which corresponds to what is traditionally
thought of as \docutilsrolecategory{vp}, might be represented as follows.
\hypertarget{subcathpsg1}{}\begin{itemize}
\item[(124)] 
\docutilsrolecategory{v}{[}\docutilsrolefeature{subcat} 〈\docutilsrolecategory{np}〉 {]}
\end{itemize}

Finally, a sentence is a kind of verbal category that has \emph{no}
requirements for further arguments, and hence has a \docutilsrolefeature{subcat}
whose value is the empty list. The \href{\#subcathpsg2}{tree (125)} shows how these
category assignments combine in a parse of \docutilsroleexample{Kim put the book on the table}.
\hypertarget{subcathpsg2}{}\begin{itemize}
\item[(125)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-36.pdf}}}\end{itemize}


%___________________________________________________________________________

\subsection{Heads Revisited}
\label{heads-revisited}

We noted in the previous section that by factoring subcategorization
information out of the main category label, we could express more
generalizations about properties of verbs. Another property of this
kind is the following: expressions of category \docutilsrolecategory{v} are heads of
phrases of category \docutilsrolecategory{vp}. Similarly (and more informally) \docutilsrolecategory{n}s are heads of \docutilsrolecategory{np}s,  \docutilsrolecategory{a}s (i.e., adjectives) are heads of \docutilsrolecategory{ap}s,  and \docutilsrolecategory{p}s (i.e., adjectives) are heads of \docutilsrolecategory{pp}s. Not all phrases have
heads — for example, it is standard to say that coordinate
phrases (e.g., \docutilsroleexample{the book and the bell}) lack heads —
nevertheless, we would like our grammar formalism to express the
mother / head-daughter
relation where it holds. Now, although it looks as though there is
something in common  between, say, \docutilsrolecategory{v} and \docutilsrolecategory{vp}, this is more
of a handy convention than a real claim, since  \docutilsrolecategory{v} and \docutilsrolecategory{vp}
formally have no more in common than \docutilsrolecategory{v} and \docutilsrolecategory{Det}.

X-bar syntax (cf. \cite{chomsky1970rn}, \cite{jackendoff1977xs}) addresses
this issue by abstracting out the notion of \index{phrasal level}\textbf{phrasal level}. It is
usual to recognize three such levels. If \docutilsrolecategory{n} represents the
lexical level, then \docutilsrolecategory{n}' represents the next level up,
corresponding to the more traditional category \docutilsrolecategory{Nom}, while
\docutilsrolecategory{n}'' represents the phrasal level, corresponding to the
category \docutilsrolecategory{np}. (The primes here replace the typographically more
demanding horizontal bars of \cite{chomsky1970rn}). \href{\#xbar0}{(126)} illustrates a
representative structure.
\hypertarget{xbar0}{}\begin{itemize}
\item[(126)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-37.pdf}}}\end{itemize}

{\noindent} The head of the structure \href{\#xbar0}{(126)} is \docutilsrolecategory{n} while \docutilsrolecategory{n}'
and \docutilsrolecategory{n}'' are called \index{(phrasal) projections}\textbf{(phrasal) projections} of \docutilsrolecategory{n}. \docutilsrolecategory{n}''
is the \index{maximal projection}\textbf{maximal projection}, and \docutilsrolecategory{n} is sometimes called the
\index{zero projection}\textbf{zero projection}. One of the central claims of X-bar syntax is
that all constituents share a structural similarity. Using \docutilsrolecategory{x} as
a variable over \docutilsrolecategory{n}, \docutilsrolecategory{v}, \docutilsrolecategory{a} and \docutilsrolecategory{p}, we say that
directly subcategorized \docutilsroleemphasis{complements} of the head are always
placed as sisters of the lexical head, whereas \docutilsroleemphasis{adjuncts} are
placed as sisters of the intermediate category, \docutilsrolecategory{x}'. Thus, the
configuration of the \docutilsrolecategory{p}'' adjunct in \href{\#xbar1}{(127)} contrasts with that
of the complement \docutilsrolecategory{p}'' in \href{\#xbar0}{(126)}.
\hypertarget{xbar1}{}\begin{itemize}
\item[(127)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-38.pdf}}}\end{itemize}

The productions in \href{\#xbar2}{(128)} illustrate how bar levels can be encoded
using feature structures.
\hypertarget{xbar2}{}\begin{itemize}
\item[(128)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{s}~→~\docutilsrolecategory{n}{[}\docutilsrolefeature{bar}=\docutilsrolefval{2}{]}~\docutilsrolecategory{v}{[}\docutilsrolefeature{bar}=\docutilsrolefval{2}{]}~\\
\docutilsrolecategory{n}{[}\docutilsrolefeature{bar}=\docutilsrolefval{2}{]}~→~\docutilsrolecategory{Det~n}{[}\docutilsrolefeature{bar}=\docutilsrolefval{1}{]}~\\
\docutilsrolecategory{n}{[}\docutilsrolefeature{bar}=\docutilsrolefval{1}{]}~→~\docutilsrolecategory{n}{[}\docutilsrolefeature{bar}=\docutilsrolefval{1}{]}~\docutilsrolecategory{p}{[}\docutilsrolefeature{bar}=\docutilsrolefval{2}{]}~\\
\docutilsrolecategory{n}{[}\docutilsrolefeature{bar}=\docutilsrolefval{1}{]}~→~\docutilsrolecategory{n}{[}\docutilsrolefeature{bar}=\docutilsrolefval{0}{]}~\docutilsrolecategory{p}{[}\docutilsrolefeature{bar}=\docutilsrolefval{2}{]}
}\end{quote}
\end{itemize}


%___________________________________________________________________________

\subsection{Auxiliary Verbs and Inversion}
\label{auxiliary-verbs-and-inversion}

Inverted clauses — where the order of subject and verb is
switched — occur in English interrogatives and also after
'negative' adverbs:
\hypertarget{inv1}{}\begin{itemize}
\item[(129)] \hypertarget{inv1a}{}\begin{itemize}
\item[a.] 
Do you like children?
\end{itemize}
\hypertarget{inv1b}{}\begin{itemize}
\item[b.] 
Can Jody walk?
\end{itemize}
\end{itemize}
\hypertarget{inv2}{}\begin{itemize}
\item[(130)] \hypertarget{inv2a}{}\begin{itemize}
\item[a.] 
Rarely do you see Kim.
\end{itemize}
\hypertarget{inv2b}{}\begin{itemize}
\item[b.] 
Never have I seen this dog.
\end{itemize}
\end{itemize}

{\noindent} However, we cannot place just any verb in pre-subject position:
\hypertarget{inv3}{}\begin{itemize}
\item[(131)] \hypertarget{inv3a}{}\begin{itemize}
\item[a.] 
*Like you children?
\end{itemize}
\hypertarget{inv3b}{}\begin{itemize}
\item[b.] 
*Walks Jody?
\end{itemize}
\end{itemize}
\hypertarget{inv4}{}\begin{itemize}
\item[(132)] \hypertarget{inv4a}{}\begin{itemize}
\item[a.] 
*Rarely see you Kim.
\end{itemize}
\hypertarget{inv4b}{}\begin{itemize}
\item[b.] 
*Never saw I this dog.
\end{itemize}
\end{itemize}

Verbs that can be positioned initially in inverted clauses belong to
the class known as \index{auxiliaries}\textbf{auxiliaries}, and as well as  \docutilsroleexample{do},
\docutilsroleexample{can} and \docutilsroleexample{have}  include \docutilsroleexample{be}, \docutilsroleexample{will}  and
\docutilsroleexample{shall}. One way of capturing such structures is with the
following production:
\hypertarget{sinv}{}\begin{itemize}
\item[(133)] \begin{quote}{\ttfamily \raggedright \noindent
\small
S{[}+inv{]}~->~V{[}+AUX{]}~NP~VP
}\end{quote}
\end{itemize}

{\noindent} That is, a clause marked as {[}\docutilsrolefeature{+inv}{]} consists of an auxiliary
verb followed by a \docutilsrolecategory{vp}. (In a more detailed grammar, we would
need to place some constraints on the form of the \docutilsrolecategory{vp}, depending
on the choice of auxiliary.) \href{\#invtree}{(134)} illustrates the structure of an
inverted clause.
\hypertarget{invtree}{}\begin{itemize}
\item[(134)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-39.pdf}}}\end{itemize}


%___________________________________________________________________________

\subsection{Unbounded Dependency Constructions}
\label{unbounded-dependency-constructions}

Consider the following contrasts:
\hypertarget{gap1}{}\begin{itemize}
\item[(135)] \hypertarget{gap1a}{}\begin{itemize}
\item[a.] 
You like Jody.
\end{itemize}
\hypertarget{gap1b}{}\begin{itemize}
\item[b.] 
*You like.
\end{itemize}
\end{itemize}
\hypertarget{gap2}{}\begin{itemize}
\item[(136)] \hypertarget{gap2a}{}\begin{itemize}
\item[a.] 
You put the card into the slot.
\end{itemize}
\hypertarget{gap2b}{}\begin{itemize}
\item[b.] 
*You put into the slot.
\end{itemize}
\hypertarget{gap2c}{}\begin{itemize}
\item[c.] 
*You put the card.
\end{itemize}
\hypertarget{gap2d}{}\begin{itemize}
\item[d.] 
*You put.
\end{itemize}
\end{itemize}

The verb \docutilsroleexample{like} requires an \docutilsrolecategory{np} complement, while
\docutilsroleexample{put} requires both a following \docutilsrolecategory{np} and \docutilsrolecategory{pp}. Examples
\href{\#gap1}{(135)} and \href{\#gap2}{(136)} show that these complements are \emph{obligatory}:
omitting them leads to ungrammaticality. Yet there are contexts in
which obligatory complements can be omitted, as \href{\#gap3}{(137)} and \href{\#gap4}{(138)}
illustrate.
\hypertarget{gap3}{}\begin{itemize}
\item[(137)] \hypertarget{gap3a}{}\begin{itemize}
\item[a.] 
Kim knows who you like.
\end{itemize}
\hypertarget{gap3b}{}\begin{itemize}
\item[b.] 
This music, you really like.
\end{itemize}
\end{itemize}
\hypertarget{gap4}{}\begin{itemize}
\item[(138)] \hypertarget{gap4a}{}\begin{itemize}
\item[a.] 
Which card do you put into the slot?
\end{itemize}
\hypertarget{gap4b}{}\begin{itemize}
\item[b.] 
Which slot do you put the card into?
\end{itemize}
\end{itemize}

{\noindent} That is, an obligatory complement can be omitted if there is an
appropriate \index{filler}\textbf{filler} in the sentence, such as the question word
\docutilsroleexample{who} in \href{\#gap3a}{(137a)}, the preposed topic \docutilsroleexample{this music} in \href{\#gap3b}{(137b)}, or
the \docutilsroleexample{wh} phrases \docutilsroleexample{which card/slot} in \href{\#gap4}{(138)}. It is common to
say that sentences like \href{\#gap3}{(137)} – \href{\#gap4}{(138)} contain \index{gaps}\textbf{gaps} where
the obligatory complements have been omitted, and these gaps are
sometimes made explicit using an underscore:
\hypertarget{gap5}{}\begin{itemize}
\item[(139)] \hypertarget{gap5a}{}\begin{itemize}
\item[a.] 
Which card do you put {\_}{\_} into the slot?
\end{itemize}
\hypertarget{gap5b}{}\begin{itemize}
\item[b.] 
Which slot do you put the card into {\_}{\_}?
\end{itemize}
\end{itemize}

{\noindent} So, a gap can occur if it is \index{licensed}\textbf{licensed} by a filler. Conversely,
fillers can only occur if there is an appropriate gap elsewhere  in
the sentence, as shown by the following examples.
\hypertarget{gap6}{}\begin{itemize}
\item[(140)] \hypertarget{gap6a}{}\begin{itemize}
\item[a.] 
*Kim knows who you like Jody.
\end{itemize}
\hypertarget{gap6b}{}\begin{itemize}
\item[b.] 
*This music, you really like hip-hop.
\end{itemize}
\end{itemize}
\hypertarget{gap7}{}\begin{itemize}
\item[(141)] \hypertarget{gap7a}{}\begin{itemize}
\item[a.] 
*Which card do you put this into the slot?
\end{itemize}
\hypertarget{gap7b}{}\begin{itemize}
\item[b.] 
*Which slot do you put the card into this one?
\end{itemize}
\end{itemize}

The mutual co-occurence between filler and gap leads to \href{\#gap3}{(137)} –
\href{\#gap4}{(138)} is sometimes termed a ''dependency``. One issue of considerable
importance in theoretical linguistics has been the nature of the
material that can intervene between a filler and the gap that it
licenses; in particular, can we simply list a finite set of strings
that separate the two? The answer is No: there is no upper bound on
the distance between filler and gap. This fact can be easily
illustrated with constructions involving sentential complements, as
shown in \href{\#gap8}{(142)}.
\hypertarget{gap8}{}\begin{itemize}
\item[(142)] \hypertarget{gap8a}{}\begin{itemize}
\item[a.] 
Who do you like {\_}{\_}?
\end{itemize}
\hypertarget{gap8b}{}\begin{itemize}
\item[b.] 
Who do you claim that you like {\_}{\_}?
\end{itemize}
\hypertarget{gap8c}{}\begin{itemize}
\item[c.] 
Who do you claim that Jody says that you like {\_}{\_}?
\end{itemize}
\end{itemize}

{\noindent} Since we can have indefinitely deep recursion of sentential
complements, the gap can be embedded indefinitely far inside the whole
sentence. This constellation of properties leads to the notion of an
\index{unbounded dependency construction}\textbf{unbounded dependency construction}; that is, a filler-gap
dependency where there is no upper bound on the distance between
filler and gap.

A variety of mechanisms have been suggested for handling unbounded
dependencies in formal grammars; we shall adopt an approach due to
Generalized Phrase Structure Grammar that involves something called
\index{slash categories}\textbf{slash categories}. A slash category is something of the form
\docutilsrolecategory{y/xp}; we interpret this as a phrase of category \docutilsrolecategory{y} that
is missing a sub-constituent of category \docutilsrolecategory{xp}. For example,
\docutilsrolecategory{s/np} is an \docutilsrolecategory{s} that is missing an \docutilsrolecategory{np}. The use of
slash categories is illustrated in \href{\#gaptree1}{(143)}.
\hypertarget{gaptree1}{}\begin{itemize}
\item[(143)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-40.pdf}}}\end{itemize}

{\noindent} The top part of the tree introduces the filler \docutilsroleexample{who} (treated as
an expression of category \docutilsrolecategory{np}{[}\docutilsrolefeature{+wh}{]}) together with a
corresponding gap-containing constituent \docutilsrolecategory{s/np}. The gap information is
then ''percolated`` down the tree via the \docutilsrolecategory{vp/np} category, until it
reaches the category \docutilsrolecategory{np/np}. At this point, the dependency
is discharged by realizing the gap information as the empty string \titlereference{e}
immediately dominated by \docutilsrolecategory{np/np}.

Do we need to think of slash categories as a completely new kind of
object in our grammars?  Fortunately, no, we don't — in fact, we
can accommodate them within our existing feature-based framework. We
do this by treating slash as a feature, and the category to its right
as a value. In other words, our ''official`` notation for \docutilsrolecategory{s/np}
will be \docutilsrolecategory{s}{[}\docutilsrolefeature{slash}=\docutilsrolefval{NP}{]}. Once we have taken this
step, it is straightforward to write a small grammar for
analyzing unbounded dependency constructions. \href{\#slashcfg}{10.4} illustrates
the main principles of slash categories, and also includes productions for
inverted clauses. To simplify presentation, we have omitted any
specification of tense on the verbs.
\hypertarget{slashcfg}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.data.show{\_}cfg(\pysrcstring{'grammars/feat1.fcfg'})
\pysrcoutput{{\%} start S}
\pysrcoutput{{\#} {\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}}
\pysrcoutput{{\#} Grammar Rules}
\pysrcoutput{{\#} {\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}}
\pysrcoutput{S{[}-INV{]} -> NP  S/NP}
\pysrcoutput{S{[}-INV{]}/?x -> NP VP/?x}
\pysrcoutput{S{[}+INV{]}/?x -> V{[}+AUX{]} NP VP/?x}
\pysrcoutput{S-BAR/?x -> Comp S{[}-INV{]}/?x}
\pysrcoutput{NP/NP ->}
\pysrcoutput{VP/?x -> V{[}SUBCAT=1, -AUX{]} NP/?x}
\pysrcoutput{VP/?x -> V{[}SUBCAT=2, -AUX{]} S-BAR/?x}
\pysrcoutput{VP/?x -> V{[}SUBCAT=3, +AUX{]} VP/?x}
\pysrcoutput{{\#} {\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}}
\pysrcoutput{{\#} Lexical Rules}
\pysrcoutput{{\#} {\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}{\#}}
\pysrcoutput{V{[}SUBCAT=1, -AUX{]} -> 'see' | 'like'}
\pysrcoutput{V{[}SUBCAT=2, -AUX{]} -> 'say' | 'claim'}
\pysrcoutput{V{[}SUBCAT=3, +AUX{]} -> 'do' | 'can'}
\pysrcoutput{NP{[}-WH{]} -> 'you' | 'children' | 'girls'}
\pysrcoutput{NP{[}+WH{]} -> 'who'}
\pysrcoutput{Comp -> 'that'}}\end{alltt}
\caption{Grammar for Simple Long-distance Dependencies}
\vspace{1ex}
\hrule
\end{pylisting}

The grammar in \href{\#slashcfg}{Listing 10.4} contains one gap-introduction production, namely
\begin{itemize}
\item[(144)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{s{[}-inv{]}}~→~\docutilsrolecategory{np}~\docutilsrolecategory{s/np}
}\end{quote}
\end{itemize}

In order to percolate the slash feature correctly, we need to add
slashes with variable values to both sides of the arrow in productions
that expand \docutilsrolecategory{s}, \docutilsrolecategory{vp} and \docutilsrolecategory{np}. For example,
\begin{itemize}
\item[(145)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{vp/?x}~→~\docutilsrolecategory{v}~\docutilsrolecategory{s-bar/?x}
}\end{quote}
\end{itemize}

{\noindent} says that a slash value can be specified on the \docutilsrolecategory{vp} mother of a
constituent if the same value is also specified on the \docutilsrolecategory{s-bar}
daughter. Finally, \href{\#empty}{(146)} allows the slash information on \docutilsrolecategory{np} to
be discharged as the empty string.
\hypertarget{empty}{}\begin{itemize}
\item[(146)] \begin{quote}{\ttfamily \raggedright \noindent
\small
\docutilsrolecategory{np/np}~→
}\end{quote}
\end{itemize}

Using \href{\#slashcfg}{10.4}, we can parse the string \docutilsroleexample{who do you claim that you
like}  into the tree shown in \href{\#gapparse}{(147)}.
\hypertarget{gapparse}{}\begin{itemize}
\item[(147)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-41.pdf}}}\end{itemize}


%___________________________________________________________________________

\subsection{Case and Gender in German}
\label{case-and-gender-in-german}

Compared with English, German has a relatively rich morphology for
agreement. For example, the definite article in German varies with
case, gender and number, as shown in \href{\#german-def-art}{Table 10.2}.
\hypertarget{german-def-art}{}
\leavevmode
\begin{longtable}[c]{|p{0.145\locallinewidth}|p{0.145\locallinewidth}|p{0.145\locallinewidth}|p{0.145\locallinewidth}|p{0.156\locallinewidth}|}
\hline

\textbf{Case}
 & 
\textbf{Masc}
 & 
\textbf{Fem}
 & 
\textbf{Neut}
 & 
\textbf{Plural}
 \\
\hline

\emph{Nom}
 & 
der
 & 
die
 & 
das
 & 
die
 \\
\hline

\emph{Gen}
 & 
des
 & 
der
 & 
des
 & 
der
 \\
\hline

\emph{Dat}
 & 
dem
 & 
der
 & 
dem
 & 
den
 \\
\hline

\emph{Acc}
 & 
den
 & 
die
 & 
das
 & 
die
 \\
\hline
\caption{
Morphological Paradigm for the German definite Article
}\end{longtable}

{\noindent} Subjects in German take the nominative case, and most verbs
govern their objects in the accusative case. However, there are
exceptions like \docutilsroleexample{helfen} that govern the dative case:
\begin{itemize}
\item[(148)] \begin{itemize}
\item[a.] 
\begin{tabular}[c]{p{0.179\locallinewidth}p{0.156\locallinewidth}p{0.121\locallinewidth}p{0.191\locallinewidth}p{0.098\locallinewidth}}

Die
 & 
Katze
 & 
sieht
 & 
den
 & 
Hund
 \\

the.NOM.FEM.SG
 & 
cat.3.FEM.SG
 & 
see.3.SG
 & 
the.ACC.MASC.SG
 & 
dog.3.MASC.SG
 \\
\multicolumn{5}{l}{
'the cat sees the dog'
} \\
\end{tabular}
\end{itemize}
\begin{itemize}
\item[b.] 
\begin{tabular}[c]{p{0.179\locallinewidth}p{0.156\locallinewidth}p{0.121\locallinewidth}p{0.191\locallinewidth}p{0.098\locallinewidth}}

*Die
 & 
Katze
 & 
sieht
 & 
dem
 & 
Hund
 \\

the.NOM.FEM.SG
 & 
cat.3.FEM.SG
 & 
see.3.SG
 & 
the.DAT.MASC.SG
 & 
dog.3.MASC.SG
 \\
\end{tabular}
\end{itemize}
\begin{itemize}
\item[c.] 
\begin{tabular}[c]{p{0.179\locallinewidth}p{0.156\locallinewidth}p{0.121\locallinewidth}p{0.191\locallinewidth}p{0.098\locallinewidth}}

Die
 & 
Katze
 & 
hilft
 & 
dem
 & 
Hund
 \\

the.NOM.FEM.SG
 & 
cat.3.FEM.SG
 & 
help.3.SG
 & 
the.DAT.MASC.SG
 & 
dog.3.MASC.SG
 \\
\multicolumn{5}{l}{
'the cat helps the dog'
} \\
\end{tabular}
\end{itemize}
\begin{itemize}
\item[d.] 
\begin{tabular}[c]{p{0.179\locallinewidth}p{0.156\locallinewidth}p{0.121\locallinewidth}p{0.191\locallinewidth}p{0.098\locallinewidth}}

*Die
 & 
Katze
 & 
hilft
 & 
den
 & 
Hund
 \\

the.NOM.FEM.SG
 & 
cat.3.FEM.SG
 & 
help.3.SG
 & 
the.ACC.MASC.SG
 & 
dog.3.MASC.SG
 \\
\end{tabular}
\end{itemize}
\end{itemize}

The grammar \href{\#germancfg}{10.5} illustrates the interaction of agreement
(comprising person, number and gender) with case.
\hypertarget{germancfg}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.data.show{\_}cfg(\pysrcstring{'grammars/german.fcfg'})
\pysrcoutput{{\%} start S}
\pysrcoutput{{\#} Grammar Rules}
\pysrcoutput{S -> NP{[}CASE=nom, AGR=?a{]} VP{[}AGR=?a{]}}
\pysrcoutput{NP{[}CASE=?c, AGR=?a{]} -> PRO{[}CASE=?c, AGR=?a{]}}
\pysrcoutput{NP{[}CASE=?c, AGR=?a{]} -> Det{[}CASE=?c, AGR=?a{]} N{[}CASE=?c, AGR=?a{]}}
\pysrcoutput{VP{[}AGR=?a{]} -> IV{[}AGR=?a{]}}
\pysrcoutput{VP{[}AGR=?a{]} -> TV{[}OBJCASE=?c, AGR=?a{]} NP{[}CASE=?c{]}}
\pysrcoutput{{\#} Lexical Rules}
\pysrcoutput{{\#} Singular determiners}
\pysrcoutput{{\#} masc}
\pysrcoutput{Det{[}CASE=nom, AGR={[}GND=masc,PER=3,NUM=sg{]}{]} -> 'der'}
\pysrcoutput{Det{[}CASE=dat, AGR={[}GND=masc,PER=3,NUM=sg{]}{]} -> 'dem'}
\pysrcoutput{Det{[}CASE=acc, AGR={[}GND=masc,PER=3,NUM=sg{]}{]} -> 'den'}
\pysrcoutput{{\#} fem}
\pysrcoutput{Det{[}CASE=nom, AGR={[}GND=fem,PER=3,NUM=sg{]}{]} -> 'die'}
\pysrcoutput{Det{[}CASE=dat, AGR={[}GND=fem,PER=3,NUM=sg{]}{]} -> 'der'}
\pysrcoutput{Det{[}CASE=acc, AGR={[}GND=fem,PER=3,NUM=sg{]}{]} -> 'die'}
\pysrcoutput{{\#} Plural determiners}
\pysrcoutput{Det{[}CASE=nom, AGR={[}PER=3,NUM=pl{]}{]} -> 'die'}
\pysrcoutput{Det{[}CASE=dat, AGR={[}PER=3,NUM=pl{]}{]} -> 'den'}
\pysrcoutput{Det{[}CASE=acc, AGR={[}PER=3,NUM=pl{]}{]} -> 'die'}
\pysrcoutput{{\#} Nouns}
\pysrcoutput{N{[}AGR={[}GND=masc,PER=3,NUM=sg{]}{]} -> 'hund'}
\pysrcoutput{N{[}CASE=nom, AGR={[}GND=masc,PER=3,NUM=pl{]}{]} -> 'hunde'}
\pysrcoutput{N{[}CASE=dat, AGR={[}GND=masc,PER=3,NUM=pl{]}{]} -> 'hunden'}
\pysrcoutput{N{[}CASE=acc, AGR={[}GND=masc,PER=3,NUM=pl{]}{]} -> 'hunde'}
\pysrcoutput{N{[}AGR={[}GND=fem,PER=3,NUM=sg{]}{]} -> 'katze'}
\pysrcoutput{N{[}AGR={[}GND=fem,PER=3,NUM=pl{]}{]} -> 'katzen'}
\pysrcoutput{{\#} Pronouns}
\pysrcoutput{PRO{[}CASE=nom, AGR={[}PER=1,NUM=sg{]}{]} -> 'ich'}
\pysrcoutput{PRO{[}CASE=acc, AGR={[}PER=1,NUM=sg{]}{]} -> 'mich'}
\pysrcoutput{PRO{[}CASE=dat, AGR={[}PER=1,NUM=sg{]}{]} -> 'mir'}
\pysrcoutput{PRO{[}CASE=nom, AGR={[}PER=2,NUM=sg{]}{]} -> 'du'}
\pysrcoutput{PRO{[}CASE=nom, AGR={[}PER=3,NUM=sg{]}{]} -> 'er' | 'sie' | 'es'}
\pysrcoutput{PRO{[}CASE=nom, AGR={[}PER=1,NUM=pl{]}{]} -> 'wir'}
\pysrcoutput{PRO{[}CASE=acc, AGR={[}PER=1,NUM=pl{]}{]} -> 'uns'}
\pysrcoutput{PRO{[}CASE=dat, AGR={[}PER=1,NUM=pl{]}{]} -> 'uns'}
\pysrcoutput{PRO{[}CASE=nom, AGR={[}PER=2,NUM=pl{]}{]} -> 'ihr'}
\pysrcoutput{PRO{[}CASE=nom, AGR={[}PER=3,NUM=pl{]}{]} -> 'sie'}
\pysrcoutput{{\#} Verbs}
\pysrcoutput{IV{[}AGR={[}NUM=sg,PER=1{]}{]} -> 'komme'}
\pysrcoutput{IV{[}AGR={[}NUM=sg,PER=2{]}{]} -> 'kommst'}
\pysrcoutput{IV{[}AGR={[}NUM=sg,PER=3{]}{]} -> 'kommt'}
\pysrcoutput{IV{[}AGR={[}NUM=pl, PER=1{]}{]} -> 'kommen'}
\pysrcoutput{IV{[}AGR={[}NUM=pl, PER=2{]}{]} -> 'kommt'}
\pysrcoutput{IV{[}AGR={[}NUM=pl, PER=3{]}{]} -> 'kommen'}
\pysrcoutput{TV{[}OBJCASE=acc, AGR={[}NUM=sg,PER=1{]}{]} -> 'sehe' | 'mag'}
\pysrcoutput{TV{[}OBJCASE=acc, AGR={[}NUM=sg,PER=2{]}{]} -> 'siehst' | 'magst'}
\pysrcoutput{TV{[}OBJCASE=acc, AGR={[}NUM=sg,PER=3{]}{]} -> 'sieht' | 'mag'}
\pysrcoutput{TV{[}OBJCASE=dat, AGR={[}NUM=sg,PER=1{]}{]} -> 'folge' | 'helfe'}
\pysrcoutput{TV{[}OBJCASE=dat, AGR={[}NUM=sg,PER=2{]}{]} -> 'folgst' | 'hilfst'}
\pysrcoutput{TV{[}OBJCASE=dat, AGR={[}NUM=sg,PER=3{]}{]} -> 'folgt' | 'hilft'}}\end{alltt}
\caption{Example Feature-Based Grammar}
\vspace{1ex}
\hrule
\end{pylisting}
\begin{description}
\item[{{\noindent} As you will see, the feature \docutilsrolefeature{objcase} is used to}] \leavevmode 
specify the case that the verb governs on its object.

\end{description}


%___________________________________________________________________________

\subsection{Exercises}
\label{id102}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Modify the grammar illustrated in  \href{\#subcatgpsg}{(120)} to
incorporate a \docutilsrolefeature{bar} feature for dealing with phrasal projections.

\item {} 
☼ Modify the German grammar in \href{\#germancfg}{10.5} to incorporate the
treatment of subcategorization presented in \href{\#sec-subcat}{10.4.1}.

\item {} 
◑ Extend the German grammar in \href{\#germancfg}{10.5} so that it can
handle so-called verb-second structures like the following:
\begin{itemize}
\item[(149)] 
Heute sieht der hund die katze.
\end{itemize}

\item {} 
★ Morphological paradigms are rarely completely regular, in
the sense of every cell in the matrix having a different
realization. For example, the present tense conjugation of the
lexeme \docutilsrolelex{walk} only has two distinct forms: \docutilsroleexample{walks} for the
3rd person singular, and \docutilsroleexample{walk} for all other combinations of
person and number. A successful analysis should not require
redundantly specifying that 5 out of the 6 possible morphological
combinations have the same realization.  Propose and implement a
method for dealing with this.

\item {} 
★ So-called \index{head features}\textbf{head features} are shared between the mother
and head daughter. For example, \docutilsrolefeature{tense} is a head feature
that is shared between a \docutilsrolecategory{vp} and its head \docutilsrolecategory{v}
daughter. See \cite{gazdar1985gps} for more details. Most of the
features we have looked at are head features — exceptions are
\docutilsrolefeature{subcat} and \docutilsrolefeature{slash}. Since the sharing of head
features is predictable, it should not need to be stated explicitly
in the grammar productions. Develop an approach that automatically
accounts for this regular behavior of head features.

\end{list}


%___________________________________________________________________________

\section{Summary}
\label{id104}
\begin{itemize}
\item {} 
The traditional categories of context-free grammar are atomic
symbols. An important motivation feature structures is to capture
fine-grained distinctions that would otherwise require a massive
multiplication of atomic categories.

\item {} 
By using variables over feature values, we can express constraints
in grammar productions that allow the realization of different feature
specifications to be inter-dependent.

\item {} 
Typically we specify fixed values of features at the lexical level
and constrain the values of features in phrases to unify with the
corresponding values in their daughters.

\item {} 
Feature values are either atomic or complex. A particular sub-case of
atomic value is the Boolean value, represented by convention as {[}+/-
\docutilsrolefeature{f}{]}.

\item {} 
Two features can share a value (either atomic or
complex). Structures with shared values are said to be
re-entrant. Shared values are represented by numerical indices (or
tags) in AVMs.

\item {} 
A path in a feature structure is a tuple of features
corresponding to the labels on  a sequence of arcs from the root of the graph
representation.

\item {} 
Two paths are equivalent if they share a value.

\item {} 
Feature structures are partially ordered by subsumption.
\docutilsrolemath{FS}$_{0}$ subsumes \docutilsrolemath{FS}$_{1}$ when
\docutilsrolemath{FS}$_{0}$ is more general (less informative) than
\docutilsrolemath{FS}$_{1}$.

\item {} 
The unification of two structures \docutilsrolemath{FS}$_{0}$ and
\docutilsrolemath{FS}$_{1}$, if successful, is the feature
structure \docutilsrolemath{FS}$_{2}$ that contains the combined
information of both \docutilsrolemath{FS}$_{0}$ and \docutilsrolemath{FS}$_{1}$.

\item {} 
If unification specializes a path π in \docutilsrolemath{FS}, then it also
specializes every path π' equivalent to π.

\item {} 
We can use feature structures to build succinct analyses of a wide
variety of linguistic phenomena, including verb subcategorization,
inversion constructions, unbounded dependency constructions and case government.

\end{itemize}


%___________________________________________________________________________

\section{Further Reading}
\label{id105}

For more examples of feature-based parsing with NLTK, please see the
guides at \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}featgram{\linebreak[0]}.{\linebreak[0]}html},
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}featstruct{\linebreak[0]}.{\linebreak[0]}html}, and
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}grammartestsuites{\linebreak[0]}.{\linebreak[0]}html}.

For an excellent introduction to the phenomenon of agreement, see
\cite{corbett2006a}.

The earliest use of features in theoretical linguistics was designed
to capture phonological properties of phonemes. For example, a sound
like /\textbf{b}/ might be decomposed into the structure {[}\docutilsrolefeature{+labial},
\docutilsrolefeature{+voice}{]}. An important motivation was to capture
generalizations across classes of segments; for example, that /\textbf{n}/ gets
realized as /\textbf{m}/ preceding any \docutilsrolefeature{+labial} consonant.
Within Chomskyan grammar, it was standard to use atomic features for
phenomena like agreement, and also to capture generalizations across
syntactic categories, by analogy with phonology.
A radical expansion of the use of features in theoretical syntax was
advocated by Generalized Phrase Structure Grammar (GPSG;
\cite{gazdar1985gps}), particularly in the use of features with complex values.

Coming more from the perspective of computational linguistics,
\cite{kay1984ug} proposed that functional aspects of language could be
captured by unification of attribute-value structures, and a similar
approach was elaborated by \cite{shieber1983fip} within the PATR-II
formalism. Early work in Lexical-Functional grammar (LFG;
\cite{kaplan1982lfg}) introduced the notion of an \index{f-structure}\textbf{f-structure} that
was primarily intended to represent the grammatical relations and
predicate-argument structure associated with a constituent structure
parse.  \cite{shieber1986iub} provides an excellent introduction to this
phase of research into feature-based grammars.

One conceptual difficulty with algebraic approaches to feature
structures arose when researchers attempted to model negation. An
alternative perspective, pioneered by \cite{kasper1986lsf} and
\cite{johnson1988avl}, argues that grammars involve \docutilsroleemphasis{descriptions} of
feature structures rather than the structures themselves. These
descriptions are combined using logical operations such as
conjunction, and negation is just the usual logical operation over
feature descriptions. This description-oriented perspective was
integral to LFG from the outset (cf. \cite{kaplan1989fal}, and was also adopted by later
versions of Head-Driven Phrase Structure Grammar (HPSG;
\cite{sag1999st}). A comprehensive bibliography of HPSG literature can be
found at \href{http://www.cl.uni-bremen.de/HPSG-Bib/}{http://www.cl.uni-bremen.de/HPSG-Bib/}.

Feature structures, as presented in this chapter, are unable to
capture important constraints on linguistic information. For example,
there is no way of saying that the only permissible values for
\docutilsrolefeature{num} are \docutilsrolefval{sg} and \docutilsrolefval{pl}, while a specification such
as {[}\docutilsrolefeature{num}=\docutilsrolefval{masc}{]} is anomalous. Similarly, we cannot say
that the complex value of \docutilsrolefeature{agr} \docutilsroleemphasis{must} contain
specifications for the features \docutilsrolefeature{per}, \docutilsrolefeature{num} and
\docutilsrolefeature{gnd}, but \docutilsroleemphasis{cannot} contain a specification such as
{[}\docutilsrolefeature{subcat}=\docutilsrolefval{3}{]}.  \index{Typed feature structures}\textbf{Typed feature structures} were developed to
remedy this deficiency. To begin with, we stipulate that feature
values are always typed. In the case of atomic values, the values just
are types. For example, we would say that the value of \docutilsrolefeature{num} is
the type \emph{num}. Moreover, \emph{num} is the most general type of value for
\docutilsrolefeature{num}. Since types are organized hierarchically, we can be more
informative by specifying the value of \docutilsrolefeature{num} is a \index{subtype}\textbf{subtype}
of \emph{num}, namely either \emph{sg} or \emph{pl}.

In the case of complex values, we say that feature structures are
themselves typed. So for example the value of \docutilsrolefeature{agr} will be a
feature structure of type \emph{agr}. We also stipulate that all and only
\docutilsrolefeature{per}, \docutilsrolefeature{num} and \docutilsrolefeature{gnd} are \index{appropriate}\textbf{appropriate} features for
a structure of type \emph{agr}.  A good early review of work on typed
feature structures is \cite{emele1990tug}. A more comprehensive examination of
the formal foundations can be found in \cite{carpenter1992ltf}, while
\cite{copestake2002itf} focuses on implementing an HPSG-oriented approach
to typed feature structures.

There is a copious literature on the analysis of German within
feature-based grammar frameworks. \cite{nerbonne1994ghd} is a good
starting point for the HPSG literature on this topic, while
\cite{mueller2002cp} gives a very extensive and detailed analysis of
German syntax in HPSG.

Chapter 15 of \cite{jurafskymartin2008} discusses feature structures,
the unification algorithm, and the integration of unification into
parsing algorithms.
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
\hypertarget{chap-semantics}{}\setcounter{chapter}{10}

%___________________________________________________________________________

\chapter{Logical Semantics}
\label{logical-semantics}
\label{chap-semantics}
% NB chapter structure isn't consistent with rest of book, re exercise subsections. 


%___________________________________________________________________________

\section{Introduction}
\label{id122}

There are many NLP applications where it would be useful to have some
representation of the \docutilsroleemphasis{meaning} of a natural language
sentence. For instance, current search engine technology
can only take us so far in giving concise and correct answers to many
questions that we might be interested in. Admittedly, Google does a
good job in answering \href{\#google1a}{(150a)}, since its first hit is \href{\#google1b}{(150b)}.
\hypertarget{google1}{}\begin{itemize}
\item[(150)] \hypertarget{google1a}{}\begin{itemize}
\item[a.] 
What is the population of Saudi Arabia?
\end{itemize}
\hypertarget{google1b}{}\begin{itemize}
\item[b.] 
Saudi Arabia - Population: 26,417,599
\end{itemize}
\end{itemize}

{\noindent}
By contrast, the result of sending \href{\#google2}{(151)} to Google is less helpful:
\hypertarget{google2}{}\begin{itemize}
\item[(151)] 
Which countries border the Mediterranean?
\end{itemize}

{\noindent}
This time, the topmost hit (and the only relevant one in the top ten)
presents the relevant information as a map of the Mediterranean
basin. Since the map is an image file, it is not easy to
extract the required list of countries from the returned page.

Even if Google succeeds in finding documents which contain information
relevant to our question, there is no guarantee that it will be in a
form which can be easily converted into an appropriate answer. One
reason for this is that the information may have to be inferred from more
than one source. This is likely to be the case when we seek an answer
to more complex questions like \href{\#google3}{(152)}:
\hypertarget{google3}{}\begin{itemize}
\item[(152)] 
Which Asian countries border the Mediterranean?
\end{itemize}

{\noindent}
Here, we would probably need to combine the results of two subqueries,
namely \href{\#google2}{(151)} and \docutilsroleexample{Which countries are in Asia?}.

The example queries we have just given are based on a paper dating
back to 1982 \cite{warren1982eea};
this describes a system, \emph{Chat-80}, which converts natural
language questions into a semantic representation, and uses the latter
to retrieve answers from a knowledge base. A knowledge base is usually
taken to be a set of sentences in some formal language; in the case of
Chat-80, it is a set of Prolog clauses. However, we can encode
knowledge in a variety of formats, including relational databases,
various kinds of graph, and first-order models. In NLTK, we have
used the third of these options to re-implement a limited version of
Chat-80:
\begin{quote}{\ttfamily \raggedright \noindent
\small
Sentence:~which~Asian~countries~border~the{\_}Mediterranean~\\
-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-{}-~\\
{\textbackslash}x.((contain(asia,~x)~{\&}~country~(x))~{\&}~border~(x,~mediterranean)~\\
set({[}'turkey',~'syria',~'israel',~'lebanon'{]})
}\end{quote}

{\noindent} As we will explain later in this chapter, a semantic
representation of the form \texttt{{\textbackslash}{\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}P{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]})} denotes a set of entities
\emph{u} that meet some condition \texttt{P{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]})}. We then ask our
knowledge base to enumerate all the entities in this set.

Let's assume more generally that knowledge is available in some
structured fashion, and that it can be interrogated by a suitable
query language. Then the challenge for NLP is to find a method for
converting natural language questions into the target query
language. An alternative paradigm for question answering is to take
something like the pages returned by a Google query as our 'knowledge
base' and then to carry out further analysis and processing of the
textual information contained in the returned pages to see whether it does
in fact provide an answer to the question. In either case, it is very
useful to be able to build a semantic representation of questions.
This NLP challenge intersects in interesting ways with one of the
key goals of linguistic theory, namely to provide a systematic
correspondence between form and meaning.

A widely adopted approach to representing meaning — or at least,
some aspects of meaning — involves translating expressions of
natural language into first-order logic (FOL). From a computational point of
view, a strong argument in favor of FOL is that it strikes a
reasonable balance between expressiveness and logical tractability. On
the one hand, it is flexible enough to represent many aspects of the
logical structure of natural language. On the other hand, automated
theorem proving for FOL has been well studied, and although
inference in FOL is not decidable, in practice many reasoning
problems are efficiently solvable using modern theorem provers
(cf. \cite{blackburn2005rin} for discussion).

While there are numerous subtle and difficult issues about how to
translate natural language constructions into FOL, we will largely ignore
these. The main focus of our discussion will be on a
different issue, namely building semantic representations which
conform to some version of the \index{Principle of Compositionality}\textbf{Principle of Compositionality}.
(See \cite{partee1995lsc} for this formulation.)
\begin{description}
\item[{Principle of Compositionality:}] \leavevmode 
The meaning of a whole is a function of the meanings of the parts
and of the way they are syntactically combined.

\end{description}

{\noindent} There is an assumption here that the semantically relevant
parts of a complex expression will be determined by a theory of
syntax. Within this chapter, we will take it for granted that
expressions are parsed against a context-free grammar. However, this
is not entailed by the Principle of Compositionality.  To summarize,
we will be concerned with the task of systematically constructing a
semantic representation in a manner that can be smoothly integrated
with the process of parsing.

The overall framework we are assuming is illustrated in \href{\#semint}{Figure (153)}. Given
a syntactic analysis of a sentence, we can build one or more semantic
representations for the sentence. Once we have a semantic
representation, we can also check whether it is true in a model.
\hypertarget{semint}{}\begin{itemize}
\item[(153)] \scalebox{0.154286}{\includegraphics{../images/semint.png}}\end{itemize}

{\noindent} A \index{model}\textbf{model} for a logical language is a set-theoretic
construction which provides a very simplified picture of how the world
is. For example, in this case, the model should contain individuals
(indicated in the diagram by small dots) corresponding to Suzie and
Fido, and it should also specify that these individuals belong to
the \docutilsrolemath{chase} relation.

The order of sections in this chapter is not what you might expect
from looking at the diagram. We will start off in the middle of \href{\#semint}{(153)} by
presenting a logical language that will provide us with
semantic representations in NLTK. Next, we will show how formulas in
the language can be systematically evaluated in a model. At the end,
we will bring everything together and describe a simple method for
constructing semantic representations as part of the parse process in
NLTK.
% This chapter will only scratch the surface of how to carry out
% semantic analysis of natural language, and will be heavily slanted
% towards the approach currently implemented within
% |NLTK|. Consequently, it needs to be read in conjunction with a good
% overall introduction to formal semantics. [Eg., B&B, Gamut, Heim?] 


%___________________________________________________________________________

\section{Propositional  Logic}
\label{propositional-logic}

The language of propositional logic represents certain aspects of
natural language, but at a high level of abstraction. The only
structure that is made explicit involves \index{logical connectives}\textbf{logical connectives};
these correspond to 'logically interesting' expressions such as
\docutilsroleexample{and} and \docutilsroleexample{not}. The basic expressions of the language are
\index{propositional variables}\textbf{propositional variables}, usually written \docutilsrolemath{p}, \docutilsrolemath{q},
\docutilsrolemath{r}, etc. Let \docutilsrolemath{A} be a finite set of such variables. There
is a disjoint set of logical connectives which contains the unary
operator ¬ (\docutilsroleexample{not}), and binary operators ∧ (\docutilsroleexample{and}),
∨ (\docutilsroleexample{or}), → (\docutilsroleexample{implies}) and ≡ (\docutilsroleexample{iff}).

The set of formulas of \titlereference{L}$_{prop}$ is described inductively:
\begin{quote}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Every element of \docutilsrolemath{A} is a formula of \titlereference{L}$_{prop}$.

\item {} 
If φ is a formula of \titlereference{L}$_{prop}$ , then so is ¬ φ.

\item {} 
If φ and ψ are formulas, then so are
(φ ∧ ψ),
(φ ∨ ψ),
(φ → ψ) and
(φ ≡ ψ).

\item {} 
Nothing else is a formula of \titlereference{L}$_{prop}$.

\end{list}
\end{quote}

{\noindent} Within \titlereference{L}$_{prop}$, we can construct formulas such as
\hypertarget{prop01}{}\begin{itemize}
\item[(154)] 
\docutilsrolemath{p} → \docutilsrolemath{q} ∨ \docutilsrolemath{r}
\end{itemize}

There are many sentences of English which could be taken to have the
logical structure shown in \href{\#prop01}{(154)}. Here's an example:
\hypertarget{prop02}{}\begin{itemize}
\item[(155)] 
If it is raining, then Kim will take an umbrella or Lee will
get wet.
\end{itemize}

In order to explain the relation between \href{\#prop01}{(154)} and \href{\#prop02}{(155)}, we need
to give a \docutilsroleemphasis{key} which maps between propositional variables and
English sentences:
\hypertarget{prop03}{}\begin{itemize}
\item[(156)] 
\docutilsrolemath{p} stands for  \docutilsroleexample{it is raining},
\docutilsrolemath{q} for \docutilsroleexample{Kim will take an umbrella} and
\docutilsrolemath{q} for \docutilsroleexample{Lee will get wet}.
\end{itemize}

The Boolean connectives of propositional logic are supported by the
\texttt{sem} package, and are parsed into various kinds of
\texttt{Expression}. We use \texttt{-}, \texttt{{\&}}, \texttt{|}, \texttt{-{\linebreak[0]}>}, \texttt{<{\linebreak[0]}-{\linebreak[0]}>} to stand,
respectively, for \docutilsroleexample{not}, \docutilsroleexample{and}, \docutilsroleexample{or}, \docutilsroleexample{implies} and
\docutilsroleexample{iff}. In the following example, we start off by creating a new
instance \texttt{lp} of the NLTK \texttt{LogicParser{\linebreak[0]}({\linebreak[0]})}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }lp = nltk.LogicParser()
\pysrcprompt{>{}>{}> }lp.parse(\pysrcstring{'-(p {\&} q)'})
\pysrcoutput{<NegatedExpression -(p {\&} q)>}
\pysrcoutput{}\pysrcprompt{>{}>{}> }lp.parse(\pysrcstring{'p {\&} q'})
\pysrcoutput{<AndExpression (p {\&} q)>}
\pysrcoutput{}\pysrcprompt{>{}>{}> }lp.parse(\pysrcstring{'p | (r -> q)'})
\pysrcoutput{<OrExpression (p | (r -> q))>}
\pysrcoutput{}\pysrcprompt{>{}>{}> }lp.parse(\pysrcstring{'p <-> -{}- p'})
\pysrcoutput{<IffExpression (p <-> -{}-p)>}}\end{alltt}

As the name suggests, propositional logic only studies the logical
structure of formulas made up of atomic propositions. We saw, for
example, that propositional variables stood for whole clauses in
English. In order to look at how predicates combine with arguments, we
need to look at a more complex language for semantic representation,
namely first-order logic. In order to show how this new language interacts with
the λ-calculus, it will be useful to introduce the notion of
types into our syntactic definition, in departure from the rather
simple approach to defining the clauses of \titlereference{L}$_{prop}$.

In the general case, we interpret sentences of a logical language
relative to a model, which is a very simplified version of the
world. A model for propositional logic needs to assign the values
\texttt{True} or \texttt{False} to every possible formula. We do this
inductively: first, every propositional variable is assigned a value,
and then we compute the value of complex formulas by consulting the
meanings of the Boolean connectives and applying them to the values of
the formula's components. Let's create a valuation:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }val1 = nltk.sem.Valuation({[}(\pysrcstring{'p'}, True), (\pysrcstring{'q'}, True), (\pysrcstring{'r'}, False){]})}\end{alltt}

We initialize a \texttt{Valuation} with a list of pairs, each of which
consists of a semantic symbol and a semantic value. The resulting
object is essentially just a dictionary that maps logical expressions
(treated as strings) to appropriate values.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }val1{[}\pysrcstring{'p'}{]}
\pysrcoutput{True}}\end{alltt}

The keys of the dictionary (sorted alphabetically) can also be
accessed via the property \texttt{symbols}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }val1.symbols
\pysrcoutput{{[}'p', 'q', 'r'{]}}}\end{alltt}

As we will see later, our models need to be somewhat more complicated
in order to handle the more complicated expressions discussed in the
next section, so for the time being, just ignore the \texttt{dom1} and
\texttt{g1} variables in the following declarations.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }dom1 = set({[}{]})
\pysrcprompt{>{}>{}> }g = nltk.sem.Assignment(dom1)}\end{alltt}

Now, let's create a model \texttt{m{\linebreak[0]} {\linebreak[0]}that{\linebreak[0]} {\linebreak[0]}uses{\linebreak[0]} {\linebreak[0]}`{\linebreak[0]}`{\linebreak[0]}val1}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }m1 = nltk.sem.Model(dom1, val1, prop=True)}\end{alltt}

The \texttt{prop{\linebreak[0]}={\linebreak[0]}True} is just a flag to say that our models are intended
for propositional logic.

Every instance of Model defines appropriate truth functions for the
Boolean connectives (and in fact they are implemented as functions
named \texttt{AND{\linebreak[0]}({\linebreak[0]})}, \texttt{IMPLIES{\linebreak[0]}({\linebreak[0]})} and so on).
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }m1.AND
\pysrcoutput{<bound method Model.AND of (set({[}{]}), {\{}'q': True, 'p': True, 'r': False{\}})>}}\end{alltt}

We can use these functions to create truth tables:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} first \pysrckeyword{in} {[}True, False{]}:
\pysrcmore{... }    \pysrckeyword{for} second \pysrckeyword{in} {[}True, False{]}:
\pysrcmore{... }        \pysrckeyword{print} \pysrcstring{"{\%}s {\%}s => {\%}s"} {\%} (first, second, m1.AND(first, second))
\pysrcoutput{True True => True}
\pysrcoutput{True False => False}
\pysrcoutput{False True => False}
\pysrcoutput{False False => False}}\end{alltt}


%___________________________________________________________________________

\section{First-Order Logic}
\label{first-order-logic}


%___________________________________________________________________________

\subsection{Predication}
\label{predication}

In first-order logic (FOL), propositions are analyzed into predicates and
arguments, which takes us a step closer to the structure of natural
languages. The standard construction rules for FOL recognize
\index{terms}\textbf{terms} such as individual variables and individual constants, and
\index{predicates}\textbf{predicates} which take differing numbers of arguments. For
example, \docutilsroleexample{Adam walks} might be formalized as \docutilsrolemathit{walk(adam)}
and \docutilsroleexample{Adam sees Betty} as \docutilsrolemathit{see(adam, betty)}. We will call
\docutilsrolemathit{walk} a \index{unary predicate}\textbf{unary predicate}, and \docutilsrolemathit{see} a \index{binary
predicate}\textbf{binary
predicate}. Semantically, \docutilsrolemathit{see} is usually modeled as a
relation, i.e., a set of pairs, and the proposition is true in a
situation just in case the ordered pair pair 〈\docutilsrolemath{a, b}〉 belongs to this set.

There is an alternative
approach in which predication is treated as function application.  In
this functional style of representation, \docutilsroleexample{Adam sees Betty} can be
formalized as \docutilsrolemathit{see(j)(m)}. That is, rather than being modeled as a
relation, \docutilsrolemathit{see} denotes a function which applies to one argument to
yield a new function that is then applied to the second argument. In
NLTK, we will in fact treat predications syntactically as function
applications, but we use a concrete syntax that allows them to
represented as \docutilsrolemath{n}-ary relations.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }parsed = lp.parse(\pysrcstring{'see(a, b)'})
\pysrcprompt{>{}>{}> }parsed.argument
\pysrcoutput{<IndividualVariableExpression b>}
\pysrcoutput{}\pysrcprompt{>{}>{}> }parsed.function
\pysrcoutput{<ApplicationExpression see(a)>}
\pysrcoutput{}\pysrcprompt{>{}>{}> }parsed.function.function
\pysrcoutput{<VariableExpression see>}}\end{alltt}

Relations are represented semantically in NLTK in the standard
set-theoretic way: as sets of tuples. For example, let's suppose we
have a domain of discourse consisting of the individuals Adam, Betty and Fido,
where Adam is a boy, Betty is a girl and Fido is a dog. For mnemonic
reasons, we use \texttt{b1}, \texttt{g1} and \texttt{d1} as the corresponding labels
in the model. We can declare the domain as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }dom2 = set({[}\pysrcstring{'b1'}, \pysrcstring{'g1'}, \pysrcstring{'d1'}{]})}\end{alltt}

As before, we are going to initialize a valuation with a list of (symbol,
value) pairs:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }v = {[}(\pysrcstring{'adam'},\pysrcstring{'b1'}),(\pysrcstring{'betty'},\pysrcstring{'g1'}),(\pysrcstring{'fido'},\pysrcstring{'d1'}),
\pysrcmore{... }(\pysrcstring{'boy'},set({[}\pysrcstring{'b1'}{]})),(\pysrcstring{'girl'},set({[}\pysrcstring{'g1'}{]})), (\pysrcstring{'dog'},set({[}\pysrcstring{'d1'}{]})),
\pysrcmore{... }(\pysrcstring{'walk'},set({[}\pysrcstring{'g1'}, \pysrcstring{'d1'}{]})),
\pysrcmore{... }(\pysrcstring{'see'}, set({[}(\pysrcstring{'b1'},\pysrcstring{'g1'}),(\pysrcstring{'d1'},\pysrcstring{'b1'}),(\pysrcstring{'g1'},\pysrcstring{'d1'}),{]})){]}
\pysrcprompt{>{}>{}> }val2 = nltk.sem.Valuation(v)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} val2
\pysrcoutput{{\{}'adam': 'b1',}
\pysrcoutput{ 'betty': 'g1',}
\pysrcoutput{ 'boy': set({[}('b1',){]}),}
\pysrcoutput{ 'dog': set({[}('d1',){]}),}
\pysrcoutput{ 'fido': 'd1',}
\pysrcoutput{ 'girl': set({[}('g1',){]}),}
\pysrcoutput{ 'see': set({[}('b1', 'g1'), ('d1', 'b1'), ('g1', 'd1'){]}),}
\pysrcoutput{ 'walk': set({[}('d1',), ('g1',){]}){\}}}}\end{alltt}

So according to this valuation, the value of \texttt{see} is a set of
tuples such that Adam sees Betty, Fido sees Adam, and
Betty sees Fido.

You may have noticed that our unary predicates (i.e, \texttt{boy}, \texttt{girl},
\texttt{dog}) also come out represented as sets of singleton tuples, rather
than just sets of individuals. This is a convenience which allows us
to have a uniform treatment of relations of any arity. In order to
combine a unary relation with an argument, we use the function
\texttt{app{\linebreak[0]}({\linebreak[0]})}. If the input relation is unary, then \texttt{app{\linebreak[0]}({\linebreak[0]})} returns a
Boolean value; if the input is \docutilsrolemathit{n}-ary, for \docutilsrolemathit{n} {\textgreater} 1,
then \texttt{app{\linebreak[0]}({\linebreak[0]})} returns an \docutilsrolemathit{n}-1-ary relation.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk.sem \pysrckeyword{import} app
\pysrcprompt{>{}>{}> }boy = val2{[}\pysrcstring{'boy'}{]}
\pysrcprompt{>{}>{}> }app(boy, \pysrcstring{'b1'})
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }app(boy, \pysrcstring{'g1'})
\pysrcoutput{False}
\pysrcoutput{}\pysrcprompt{>{}>{}> }see = val2{[}\pysrcstring{'see'}{]}
\pysrcprompt{>{}>{}> }app(see, \pysrcstring{'g1'})
\pysrcoutput{set({[}('d1',){]})}
\pysrcoutput{}\pysrcprompt{>{}>{}> }app(app(see, \pysrcstring{'g1'}), \pysrcstring{'d1'})
\pysrcoutput{True}}\end{alltt}


%___________________________________________________________________________

\subsection{Individual Variables and Assignments}
\label{individual-variables-and-assignments}

In FOL, arguments of predicates can also be individual variables
such as \texttt{x}, \texttt{y} and \texttt{z}. These can be thought of as similar to
personal pronouns like \docutilsroleexample{he}, \docutilsroleexample{she} and \docutilsroleexample{it}, in that we
need to know about the context of use in order to figure out their
denotation. In our models, the counterpart of a context of use is a
variable \index{Assignment}\textbf{Assignment}. This is a mapping from individual variables to
entities in the domain.
Assignments are created using the \texttt{Assignment} constructor, which
also takes the model's domain of discourse as a parameter. We are not
required to actually enter any bindings, but if we do, they are in a
(variable, value) format similar to what we say earlier for valuations.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }g = nltk.sem.Assignment(dom2, {[}(\pysrcstring{'x'}, \pysrcstring{'b1'}), (\pysrcstring{'y'}, \pysrcstring{'d1'}){]})
\pysrcprompt{>{}>{}> }g
\pysrcoutput{{\{}'y': 'd1', 'x': 'b1'{\}}}}\end{alltt}

{\noindent} In addition, there is a \texttt{\pysrckeyword{print}({\linebreak[0]})} format for assignments which
uses a notation closer to that in logic textbooks:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} g
\pysrcoutput{g{[}d1/y{]}{[}b1/x{]}}}\end{alltt}

Let's now look at how we can evaluate an atomic formula of
FOL. First, we create a model, then we use the \texttt{evaluate{\linebreak[0]}({\linebreak[0]})} method
to compute the truth value.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }m2 = nltk.sem.Model(dom2, val2)
\pysrcprompt{>{}>{}> }m2.evaluate(\pysrcstring{'see(betty, y)'}, g)
\pysrcoutput{True}}\end{alltt}

What's happening here? Essentially, we are making a call to
\texttt{app{\linebreak[0]}({\linebreak[0]}app{\linebreak[0]}({\linebreak[0]}see{\linebreak[0]},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}g1{\linebreak[0]}'}){\linebreak[0]},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}d1{\linebreak[0]}'})} just as in our earlier example.
However, when the interpretation function encounters the variable \texttt{\pysrcstring{'{\linebreak[0]}y{\linebreak[0]}'}},
rather than checking for a value in \texttt{val2}, it asks the variable
assignment \texttt{g} to come up with a value:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }g{[}\pysrcstring{'y'}{]}
\pysrcoutput{'d1'}}\end{alltt}

Since we already know that 'b1' and 'g1' stand in the \docutilsrolemathit{see}
relation, the value \texttt{True} is what we expected. In this case, we can
say that assignment \texttt{g} \index{satisfies}\textbf{satisfies} the formula 'see(adam, y)'.
By contrast, the following formula evaluates to \texttt{False} relative to
\texttt{g} — check that you see why this is.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }m2.evaluate(\pysrcstring{'see(x, y)'}, g)
\pysrcoutput{False}}\end{alltt}

In our approach (though not in standard first-order logic), variable assignments
are \docutilsroleemphasis{partial}. For example, \texttt{g} says nothing about any variables
apart from \texttt{\pysrcstring{'{\linebreak[0]}x{\linebreak[0]}'}} and \texttt{\pysrcstring{'{\linebreak[0]}y{\linebreak[0]}'}\pysrcstring{'{\linebreak[0]}'}.{\linebreak[0]} {\linebreak[0]}The{\linebreak[0]} {\linebreak[0]}method{\linebreak[0]} {\linebreak[0]}`{\linebreak[0]}`{\linebreak[0]}purge{\linebreak[0]}({\linebreak[0]})} clears all
bindings from an assignment.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }g.purge()
\pysrcprompt{>{}>{}> }g
\pysrcoutput{{\{}{\}}}}\end{alltt}

If we now try to evaluate a formula such as 'see(adam, y)' relative to
\texttt{g}, it is like trying to interpret a sentence containing a \docutilsroleexample{she} when
we don't know what \docutilsroleexample{she} refers to. In this case, the evaluation function
fails to deliver a truth value.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }m2.evaluate(\pysrcstring{'see(adam, y)'}, g)
\pysrcoutput{'Undefined'}}\end{alltt}


%___________________________________________________________________________

\subsection{Quantification and Scope}
\label{quantification-and-scope}

First-order logic standardly offers us two quantifiers, \docutilsroleexample{all} (or \docutilsroleexample{every})
and \docutilsroleexample{some}. These are formally written as ∀ and ∃,
respectively. At the syntactic level, quantifiers are used to bind
individual variables like \texttt{\pysrcstring{'{\linebreak[0]}x{\linebreak[0]}'}} and \texttt{\pysrcstring{'{\linebreak[0]}y{\linebreak[0]}'}}. The following two sets
of examples show a simple English example, a logical representation,
and the encoding which is accepted by the NLTK \texttt{logic} module.
\hypertarget{forall1}{}\begin{itemize}
\item[(157)] \hypertarget{forall1a}{}\begin{itemize}
\item[a.] 
Every dog barks.
\end{itemize}
\hypertarget{forall1b}{}\begin{itemize}
\item[b.] 
∀\docutilsrolemathit{x.(dog(x)} → \docutilsrolemathit{bark(x))}
\end{itemize}
\hypertarget{forall1c}{}\begin{itemize}
\item[c.] 
\texttt{all{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}-{\linebreak[0]}>{\linebreak[0]} {\linebreak[0]}bark{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]})}
\end{itemize}
\end{itemize}
\hypertarget{exists1}{}\begin{itemize}
\item[(158)] \begin{itemize}
\item[a.] 
Some girl walks.
\end{itemize}
\begin{itemize}
\item[b.] 
∃\docutilsrolemathit{x.(girl(x)} ∧ \docutilsrolemathit{walk(x))}
\end{itemize}
\hypertarget{existsc}{}\begin{itemize}
\item[c.] 
\texttt{some{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}girl{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\&}{\linebreak[0]} {\linebreak[0]}walk{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]})}
\end{itemize}
\end{itemize}

In the \href{\#existsc}{(158c)}, the quantifier \texttt{some} binds both occurences of the
variable \texttt{\pysrcstring{'{\linebreak[0]}x{\linebreak[0]}'}}. As a result, \href{\#existsc}{(158c)} is said to be a \index{closed
formula}\textbf{closed
formula}. By contrast, if we look at the body of \href{\#existsc}{(158c)},
the variables are unbound:
\hypertarget{exists2}{}\begin{itemize}
\item[(159)] 
girl(x) {\&} walk(x)
\end{itemize}

\href{\#exists2}{(159)} is said to be an \index{open formula}\textbf{open formula}. As we saw earlier, the
interpretation of open formulas depends on the particular variable
assignment that we are using.

One of the crucial insights of modern
logic is that the notion of variable satisfaction can be used to
provide an interpretation to quantified formulas. Let's continue to
use \href{\#existsc}{(158c)} as an example. When is it true? Let's think about all the
individuals in our domain, i.e., in \texttt{dom2}. We want to check whether
any of these individuals have the property of being a girl and
walking. In other words, we want to know if there is some \texttt{u} in
\texttt{dom2} such that \texttt{g{\linebreak[0]}{[}{\linebreak[0]}u{\linebreak[0]}/{\linebreak[0]}x{\linebreak[0]}{]}} satisfies the open formula
\href{\#exists2}{(159)}. Consider the following:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }m2.evaluate(\pysrcstring{'some x.(girl(x) {\&} walk(x))'}, g)
\pysrcoutput{True}}\end{alltt}

\texttt{evaluate{\linebreak[0]}({\linebreak[0]})} returns \texttt{True} here because there is some \texttt{u} in
\texttt{dom2} such that \href{\#exists2}{(159)} is satisfied by an assigment which binds
\texttt{\pysrcstring{'{\linebreak[0]}x{\linebreak[0]}'}} to \texttt{u}. In fact, \texttt{g1} is such a \texttt{u}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }m2.evaluate(\pysrcstring{'girl(x) {\&} walk(x)'}, g.add(\pysrcstring{'x'}, \pysrcstring{'g1'}))
\pysrcoutput{True}}\end{alltt}

One useful tool offered by NLTK is the \texttt{satisfiers{\linebreak[0]}({\linebreak[0]})} method. This
lists all the individuals that satisfy an open formula. The method
parameters are a parsed formula, a variable, and an assignment. Here
are a few examples:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fmla1 = lp.parse(\pysrcstring{'girl(x) | boy(x)'})
\pysrcprompt{>{}>{}> }m2.satisfiers(fmla1, \pysrcstring{'x'}, g)
\pysrcoutput{set({[}'b1', 'g1'{]})}
\pysrcoutput{}\pysrcprompt{>{}>{}> }fmla2 = lp.parse(\pysrcstring{'girl(x) -> walk(x)'})
\pysrcprompt{>{}>{}> }m2.satisfiers(fmla2, \pysrcstring{'x'}, g)
\pysrcoutput{set({[}'b1', 'g1', 'd1'{]})}
\pysrcoutput{}\pysrcprompt{>{}>{}> }fmla3 = lp.parse(\pysrcstring{'walk(x) -> girl(x)'})
\pysrcprompt{>{}>{}> }m2.satisfiers(fmla3, \pysrcstring{'x'}, g)
\pysrcoutput{set({[}'b1', 'g1'{]})}}\end{alltt}

It's useful to think about why \texttt{fmla2} and \texttt{fmla3} receive the
values they do. In particular, recall the truth conditions for \texttt{-{\linebreak[0]}>}
(encoded via the function \texttt{IMPLIES{\linebreak[0]}({\linebreak[0]})} in every model):
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} first \pysrckeyword{in} {[}True, False{]}:
\pysrcmore{... }    \pysrckeyword{for} second \pysrckeyword{in} {[}True, False{]}:
\pysrcmore{... }        \pysrckeyword{print} \pysrcstring{"{\%}s {\%}s => {\%}s"} {\%} (first, second, m2.IMPLIES(first, second))
\pysrcoutput{True True => True}
\pysrcoutput{True False => False}
\pysrcoutput{False True => True}
\pysrcoutput{False False => True}}\end{alltt}

This means that
\texttt{fmla2} is equivalent to this:
\hypertarget{fmla2or}{}\begin{itemize}
\item[(160)] 
\texttt{-{\linebreak[0]} {\linebreak[0]}girl{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}|{\linebreak[0]} {\linebreak[0]}walk{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]})}
\end{itemize}

That is, \href{\#fmla2or}{(160)} is satisfied by something which either isn't a girl
or walks. Since neither \texttt{b1} (Adam) nor \texttt{d1} (Fido)
are girls, according to model \texttt{m2}, they both satisfy
the whole formula. And of course \texttt{g1} satisfies the formula because \texttt{g1}
satisfies both disjuncts. Now, since every member of the domain of
discourse satisfies \texttt{fmla2}, the corresponding universally
quantified formula is also true.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }m2.evaluate(\pysrcstring{'all x.(girl(x) -> walk(x))'}, g)
\pysrcoutput{True}}\end{alltt}

In other words, a universally quantified formula ∀\docutilsrolemathit{x.}φ is true with respect to \texttt{g} just in case for
every \texttt{u}, φ is true with respect to \texttt{g{\linebreak[0]}{[}{\linebreak[0]}u{\linebreak[0]}/{\linebreak[0]}x{\linebreak[0]}{]}}.
% |nopar| The inclusion of first-order quantifiers motivates the final clause of
% the definition of our version of |fol|.
% 
% 7. If `x`:mathit: |element|  **Var**\ (**Ind**) and |phi|
%    |element|  **Term**\ (**Bool**), then |forall|\ `x.`:mathit:\
%    |phi|, |exists|\ `x.`:mathit:\ |phi|  |element|  **Term**\
%  (**Bool**).
% 
%  One important property of forall1b_ often trips people up. The logical
%  rendering in effect says that *if* something is a dog, then it barks,
%  but makes no commitment to the existence of dogs. So in a situation
%  where nothing is a dog, forall1b_ will still come out true. (Remember
%  that ``'(p implies q)'`` is true when ``'p'`` is false.) Now you might
%  argue that forall1b_ does presuppose the existence of dogs,
%  and that the logic formalization is wrong.
%  But it is possible  to find other
%  examples which lack such a presupposition.  For instance, we might
%  explain that the value of the Python expression ``re.sub('ate', '8',
%  astring)`` is the result of replacing all occurrences of ``'ate'`` in
%  ``astring`` by ``'8'``, even though there may in fact be no such occurrences. 


%___________________________________________________________________________

\subsection{Quantificatier Scope Ambiguity}
\label{quantificatier-scope-ambiguity}

What happens when we want to give a formal representation of a
sentence with \emph{two} quantifiers, such as the following?
\hypertarget{scope1}{}\begin{itemize}
\item[(161)] 
Every girl chases a dog.
\end{itemize}

There are (at least) two ways of expressing \href{\#scope1}{(161)} in FOL:
\hypertarget{scope2}{}\begin{itemize}
\item[(162)] \hypertarget{scope2a}{}\begin{itemize}
\item[a.] 
∀\docutilsrolemathit{x.((girl x)} → ∃\docutilsrolemathit{y.((dog y)} ∧ \docutilsrolemathit{(chase y x)))}
\end{itemize}
\hypertarget{scope2b}{}\begin{itemize}
\item[b.] 
∃\docutilsrolemathit{y.((dog y)} ∧ ∀\docutilsrolemathit{x.((every x)} →  \docutilsrolemathit{(chase y x)))}
\end{itemize}
\end{itemize}

Can we use both of these? Then answer is Yes, but they have different
meanings. \href{\#scope2b}{(162b)} is logically stronger than \href{\#scope2a}{(162a)}: it claims that
there is a unique dog, say Fido, which is chased by every girl.
\href{\#scope2a}{(162a)}, on the other hand, just requires that for every girl
\docutilsrolemathit{g}, we can find some dog which \docutilsrolemathit{d} chases; but this could
be a different dog in each case. We distinguish between \href{\#scope2a}{(162a)} and
\href{\#scope2b}{(162b)} in terms of the \index{scope}\textbf{scope} of the quantifiers. In the first,
∀ has wider scope than ∃, while in \href{\#scope2b}{(162b)}, the scope ordering
is reversed. So now we have two ways of representing the meaning of
\href{\#scope1}{(161)}, and they are both quite legitimate. In other words, we are
claiming that \href{\#scope1}{(161)} is \emph{ambiguous} with respect to quantifier scope,
and the formulas in \href{\#scope2}{(162)} give us a formal means of making the two
readings explicit. However, we are not just interested in associating
two distinct representations with \href{\#scope1}{(161)}. We also want to show in
detail how the two representations lead to different conditions for
truth in a formal model.

{\noindent} In order to examine the ambiguity more closely, let's fix our
valuation as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }v3 = {[}(\pysrcstring{'john'}, \pysrcstring{'b1'}),
\pysrcmore{... }(\pysrcstring{'mary'}, \pysrcstring{'g1'}),
\pysrcmore{... }(\pysrcstring{'suzie'}, \pysrcstring{'g2'}),
\pysrcmore{... }(\pysrcstring{'fido'}, \pysrcstring{'d1'}),
\pysrcmore{... }(\pysrcstring{'tess'}, \pysrcstring{'d2'}),
\pysrcmore{... }(\pysrcstring{'noosa'}, \pysrcstring{'n'}),
\pysrcmore{... }(\pysrcstring{'girl'}, set({[}\pysrcstring{'g1'}, \pysrcstring{'g2'}{]})),
\pysrcmore{... }(\pysrcstring{'boy'}, set({[}\pysrcstring{'b1'}, \pysrcstring{'b2'}{]})),
\pysrcmore{... }(\pysrcstring{'dog'}, set({[}\pysrcstring{'d1'}, \pysrcstring{'d2'}{]})),
\pysrcmore{... }(\pysrcstring{'bark'}, set({[}\pysrcstring{'d1'}, \pysrcstring{'d2'}{]})),
\pysrcmore{... }(\pysrcstring{'walk'}, set({[}\pysrcstring{'b1'}, \pysrcstring{'g2'}, \pysrcstring{'d1'}{]})),
\pysrcmore{... }(\pysrcstring{'chase'}, set({[}(\pysrcstring{'b1'}, \pysrcstring{'g1'}), (\pysrcstring{'b2'}, \pysrcstring{'g1'}), (\pysrcstring{'g1'}, \pysrcstring{'d1'}), (\pysrcstring{'g2'}, \pysrcstring{'d2'}){]})),
\pysrcmore{... }(\pysrcstring{'see'}, set({[}(\pysrcstring{'b1'}, \pysrcstring{'g1'}), (\pysrcstring{'b2'}, \pysrcstring{'d2'}), (\pysrcstring{'g1'}, \pysrcstring{'b1'}),
\pysrcmore{... }(\pysrcstring{'d2'}, \pysrcstring{'b1'}), (\pysrcstring{'g2'}, \pysrcstring{'n'}){]})),
\pysrcmore{... }(\pysrcstring{'in'}, set({[}(\pysrcstring{'b1'}, \pysrcstring{'n'}), (\pysrcstring{'b2'}, \pysrcstring{'n'}), (\pysrcstring{'d2'}, \pysrcstring{'n'}){]})),
\pysrcmore{... }(\pysrcstring{'with'}, set({[}(\pysrcstring{'b1'}, \pysrcstring{'g1'}), (\pysrcstring{'g1'}, \pysrcstring{'b1'}), (\pysrcstring{'d1'}, \pysrcstring{'b1'}), (\pysrcstring{'b1'}, \pysrcstring{'d1'}){]})){]}
\pysrcprompt{>{}>{}> }val3 = nltk.sem.Valuation(v3)}\end{alltt}

{\noindent} We can use the graph in \href{\#chasegraph}{(163)} to visualize the
\docutilsrolemathit{chase} relation.
\hypertarget{chasegraph}{}\begin{itemize}
\item[(163)] \scalebox{0.154286}{\includegraphics{../images/models_chase.png}}\end{itemize}

{\noindent} In \href{\#chasegraph}{(163)}, an arrow between two individuals \docutilsrolemath{x} and
\docutilsrolemath{y} indicates that \docutilsrolemath{x} chases
\docutilsrolemath{y}. So \texttt{b1} and \texttt{b2} both chase \texttt{g1}, while \texttt{g1} chases
\texttt{d1} and \texttt{g2} chases \texttt{d2}. In this model, formula \texttt{scope2a{\_}} above
is true but \texttt{scope2b{\_}} is false. One way of exploring these results is by
using the \texttt{satisfiers{\linebreak[0]}({\linebreak[0]})} method of \texttt{Model} objects.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }dom3 = val3.domain
\pysrcprompt{>{}>{}> }m3 = nltk.sem.Model(dom3, val3)
\pysrcprompt{>{}>{}> }g = nltk.sem.Assignment(dom3)
\pysrcprompt{>{}>{}> }fmla1 = lp.parse(\pysrcstring{'(girl(x) -> exists y.(dog(y) and chase(x, y)))'})
\pysrcprompt{>{}>{}> }m3.satisfiers(fmla1, \pysrcstring{'x'}, g)
\pysrcoutput{set({[}'g2', 'g1', 'n', 'b1', 'b2', 'd2', 'd1'{]})}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent} This gives us the set of individuals that can be assigned as the value
of \texttt{x} in \texttt{fmla1}. In particular, every girl is included in this
set. By contrast, consider the formula \texttt{fmla2} below; this has no
satisfiers for the variable \texttt{y}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fmla2 = lp.parse(\pysrcstring{'(dog(y) {\&} all x.(girl(x) -> chase(x, y)))'})
\pysrcprompt{>{}>{}> }m3.satisfiers(fmla2, \pysrcstring{'y'}, g)
\pysrcoutput{set({[}{]})}
\pysrcoutput{}\pysrcprompt{>{}>{}>}}\end{alltt}

{\noindent} That is, there is no dog that is chased by both \texttt{g1} and
\texttt{g2}. Taking a slightly different open formula, \texttt{fmla3}, we
can verify that there is a girl, namely \texttt{g1}, who is chased by every boy.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fmla3 = lp.parse(\pysrcstring{'(girl(y) {\&} all x.(boy(x) -> chase(x, y)))'})
\pysrcprompt{>{}>{}> }m3.satisfiers(fmla3, \pysrcstring{'y'}, g)
\pysrcoutput{set({[}'g1'{]})}}\end{alltt}
% ADD something about the parse_valuation method 


%___________________________________________________________________________

\section{Evaluating English Sentences}
\label{evaluating-english-sentences}


%___________________________________________________________________________

\subsection{Using the \texttt{sem} Feature}
\label{using-the-sem-feature}

Until now, we have taken for granted that we have some appropriate
logical formulas to interpret. However, ideally we would like to
derive these formulas from natural language input. One relatively easy
way of achieving this goal is to build on the grammar framework
developed in \href{\#chap-featgram}{Chapter 10}. Our first step is to introduce a new feature,
\texttt{sem}. Because values of \texttt{sem} generally need to be treated
differently from other feature values, we use the convention of
enclosing them in angle brackets. \href{\#sem1}{(164)} illustrates a first
approximation to the kind of analyses we would like to build.
\hypertarget{sem1}{}\begin{itemize}
\item[(164)] \raisebox{-\height}{\scalebox{0.771429}{\includegraphics{tree_images/book-tree-42.pdf}}}\end{itemize}

{\noindent} Thus, the \texttt{sem} value at the root node shows a semantic
representation for the whole sentence, while the \texttt{sem} values at
lower nodes show semantic representations for constituents of the
sentence. So far, so good, but how do we write grammar rules which
will give us this kind of result? To be more specific, suppose we have
a \docutilsrolecategory{np} and \docutilsrolecategory{vp} constituents with appropriate values for their
\texttt{sem} nodes? If you reflect on the machinery that was introduced in
discussing the λ calculus, you might guess that function
application will be central to composing semantic values. You will
also remember that our feature-based grammar framework gives us the
means to refer to \docutilsroleemphasis{variable} values. Putting this together, we can
postulate a rule like \href{\#sem2}{(165)} for building the \texttt{sem} value of an
\docutilsrolecategory{s}.  (Observe that in the case where the value of \texttt{sem} is a
variable, we omit the angle brackets.)
\hypertarget{sem2}{}\begin{itemize}
\item[(165)] \begin{quote}{\ttfamily \raggedright \noindent
\small
S{[}sem~=~<app(?vp,?subj)>{]}~->~NP{[}sem=?subj{]}~VP{[}sem=?vp{]}
}\end{quote}
\end{itemize}

\href{\#sem2}{(165)} tells us that given some \texttt{sem} value \texttt{?{\linebreak[0]}subj} for the subject
\docutilsrolecategory{np} and some \texttt{sem} value \texttt{?{\linebreak[0]}vp} for the \docutilsrolecategory{vp}, the \texttt{sem}
value of the \docutilsrolecategory{s} mother is constructed by applying \texttt{?{\linebreak[0]}vp} as a
functor to \texttt{?{\linebreak[0]}np}.  From this, we can conclude that \texttt{?{\linebreak[0]}vp} has to
denote a function which has the denotation of \texttt{?{\linebreak[0]}np} in its
domain; in fact, we are going to assume that \texttt{?{\linebreak[0]}vp} denotes a
curryed characteristic function on individuals. \href{\#sem2}{(165)} is a nice example of
building semantics using \index{the principle of compositionality}\textbf{the principle of compositionality}:
that is, the principle that the semantics of a complex expression is a
function of the semantics of its parts.

To complete the grammar is very straightforward; all we require are the
rules shown in \href{\#sem3}{(166)}.
\hypertarget{sem3}{}\begin{itemize}
\item[(166)] \begin{quote}{\ttfamily \raggedright \noindent
\small
VP{[}sem=?v{]}~->~IV{[}sem=?v{]}~\\
NP{[}sem=<jane>{]}~->~'Jane'~\\
IV{[}sem=<walk>{]}~->~'walks'
}\end{quote}
\end{itemize}

{\noindent}
The \docutilsrolecategory{vp} rule says that the mother's semantics is the same as the
head daughter's. The two lexical rules just introduce non-logical
constants to serve as the semantic values of \docutilsroleexample{Jane} and
\docutilsroleexample{walks} respectively. This grammar can be parsed using the chart
parser in \texttt{parse{\linebreak[0]}.{\linebreak[0]}featurechart}, and the trace in \href{\#sem4}{(167)}
shows how semantic values are derived by feature unification in the
process of building a parse tree.
\hypertarget{sem4}{}\begin{itemize}
\item[(167)] \begin{quote}{\ttfamily \raggedright \noindent
\small
Predictor~|>~.~.|~S{[}sem='(?vp~?subj)'{]}~->~*~NP{[}sem=?subj{]}~VP{[}sem=?vp{]}~\\
Scanner~~~|{[}-{]}~.|~{[}0:1{]}~'Jane'~\\
Completer~|{[}->~.|~S{[}sem='(?vp~john)'{]}~->~NP{[}sem='john'{]}~*~VP{[}sem=?vp{]}~\\
Predictor~|.~>~.|~VP{[}sem=?v{]}~->~*~IV{[}sem=?v{]}~\\
Scanner~~~|.~{[}-{]}|~{[}1:2{]}~'walks'~\\
Completer~|.~{[}-{]}|~VP{[}sem='walk'{]}~->~IV{[}sem='walk'{]}~*~\\
Completer~|{[}==={]}|~S{[}sem='(walk~john)'{]}~->~NP{[}sem='john'{]}~VP{[}sem='walk'{]}~*~\\
Completer~|{[}==={]}|~{[}INIT{]}~->~S~*
}\end{quote}
\end{itemize}


%___________________________________________________________________________

\subsection{Quantified \docutilsrolecategory{np}s}
\label{quantified-nps}

You might be thinking this is all too easy — surely there is a
bit more to building compositional semantics. What about quantifiers,
for instance? Right, this is a crucial issue. For example, we want
\href{\#sem5a}{(168a)} to be given a semantic representation like \href{\#sem5b}{(168b)}. How can this
be accomplished?
\hypertarget{sem5}{}\begin{itemize}
\item[(168)] \hypertarget{sem5a}{}\begin{itemize}
\item[a.] 
A dog barks.
\end{itemize}
\hypertarget{sem5b}{}\begin{itemize}
\item[b.] 
\texttt{\pysrcstring{'{\linebreak[0]}exists{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\&}{\linebreak[0]} {\linebreak[0]}({\linebreak[0]}bark{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\end{itemize}
\end{itemize}

{\noindent} Let's make the assumption that our \docutilsroleemphasis{only} operation for building
complex semantic representations is function
application. Then our problem is this: how do we give a semantic
representation to quantified \docutilsrolecategory{np}s such as \docutilsroleexample{a dog} so that
they can be combined with something like \texttt{\pysrcstring{'{\linebreak[0]}walk{\linebreak[0]}'}} to give a result
like \href{\#sem5b}{(168b)}? As a first step, let's make the subject's \texttt{sem} value
act as the functor rather than the argument. Now we are
looking for way of instantiating \texttt{?{\linebreak[0]}np} so that \href{\#sem6a}{(169a)} is equivalent
to \href{\#sem6b}{(169b)}.
\hypertarget{sem6}{}\begin{itemize}
\item[(169)] \hypertarget{sem6a}{}\begin{itemize}
\item[a.] 
\begin{lineblock}{0em}
\item[] {[}sem={\textless}bark(?np){\textgreater}{]}
\end{lineblock}
\end{itemize}
\hypertarget{sem6b}{}\begin{itemize}
\item[b.] 
\begin{lineblock}{0em}
\item[] {[}sem={\textless}exist x.(dog(x) {\&} bark(x)){\textgreater}{]}
\end{lineblock}
\end{itemize}
\end{itemize}

{\noindent} This is where λ abstraction comes to the rescue;
doesn't \href{\#sem6}{(169)} look a bit reminiscent of carrying out β-reduction
in the λ-calculus? In other words, we want a λ term
\docutilsrolemathit{M} to replace \texttt{\pysrcstring{'{\linebreak[0]}?{\linebreak[0]}np{\linebreak[0]}'}} so that applying \docutilsrolemathit{M} to
\texttt{\pysrcstring{'{\linebreak[0]}bark{\linebreak[0]}'}} yields \href{\#sem5b}{(168b)}.  To do this, we replace the occurence of
\texttt{\pysrcstring{'{\linebreak[0]}bark{\linebreak[0]}'}} in \href{\#sem5b}{(168b)} by a variable \texttt{\pysrcstring{'{\linebreak[0]}P{\linebreak[0]}'}}, and bind the variable with
λ, as shown in \href{\#sem7}{(170)}.
\hypertarget{sem7}{}\begin{itemize}
\item[(170)] 
\texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}P{\linebreak[0]}.{\linebreak[0]}exists{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\&}{\linebreak[0]} {\linebreak[0]}P{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\end{itemize}

{\noindent} As a point of interest, we have used a different style of variable in
\href{\#sem7}{(170)}, that is \texttt{\pysrcstring{'{\linebreak[0]}P{\linebreak[0]}'}} rather than \texttt{\pysrcstring{'{\linebreak[0]}x{\linebreak[0]}'}} or \texttt{\pysrcstring{'{\linebreak[0]}y{\linebreak[0]}'}}. This is to signal
that we are abstracting over a different kind of thing — not an
individual, but a function from \textbf{Ind} to \textbf{Bool}. So the type of
\href{\#sem7}{(170)} as a whole is ((\textbf{Ind} → \textbf{Bool}) → \textbf{Bool}). We
will take this to be the type of \docutilsrolecategory{np}s in general. To illustrate
further, a universally quantified \docutilsrolecategory{np} will look like \href{\#sem7univ}{(171)}.
\hypertarget{sem7univ}{}\begin{itemize}
\item[(171)] 
\texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}P{\linebreak[0]}.{\linebreak[0]}all{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}-{\linebreak[0]}>{\linebreak[0]} {\linebreak[0]}P{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\end{itemize}

We are pretty much done now, except that we also want to carry out a
further abstraction plus application for the process of combining the
semantics of the determiner \docutilsroleexample{a} with the semantics of \docutilsroleexample{dog}.
Applying \href{\#sem7}{(170)} as a functor to \texttt{\pysrcstring{'{\linebreak[0]}bark{\linebreak[0]}'}} gives us \texttt{\pysrcstring{'{\linebreak[0]}({\linebreak[0]}{\textbackslash}{\linebreak[0]}P{\linebreak[0]}.{\linebreak[0]}exists{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}dog}
\pysrcstring{({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\&}{\linebreak[0]} {\linebreak[0]}P{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]} {\linebreak[0]}bark{\linebreak[0]}){\linebreak[0]}'}}, and carrying out β-reduction yields
just what we wanted, namely \href{\#sem5b}{(168b)}.

NLTK provides some utilities to make it easier to derive and inspect
semantic interpretations. \texttt{text{\_}interpret{\linebreak[0]}({\linebreak[0]})} is intended for batch
interpretation of a list of input sentences. It builds a dictionary
\texttt{d} where for each sentence \texttt{sent} in the input,
\texttt{d{\linebreak[0]}{[}{\linebreak[0]}sent{\linebreak[0]}{]}} is a list of paired trees and semantic representations for
\texttt{sent}. The value is a list, since \texttt{sent} may be syntactically
ambiguous; in the following example, we just look at the first member
of the list.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }grammar = nltk.data.load(\pysrcstring{'grammars/sem1.fcfg'})
\pysrcprompt{>{}>{}> }result  = nltk.sem.text{\_}interpret({[}\pysrcstring{'a dog barks'}{]}, grammar, beta{\_}reduce=0)
\pysrcprompt{>{}>{}> }(syntree, semrep) = result{[}\pysrcstring{'a dog barks'}{]}{[}0{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{print} syntree
\pysrcoutput{(S{[}sem=<exists x.(dog(x) {\&} bark(x))>{]}}
\pysrcoutput{  (NP{[}sem=<{\textbackslash}P.exists x.(dog(x) {\&} P(x))>{]}}
\pysrcoutput{    (Det{[}sem=<{\textbackslash}Q.{\textbackslash}P.exists x.(Q(x) {\&} P(x))>{]} a)}
\pysrcoutput{     (N{[}sem=<dog>{]} dog))}
\pysrcoutput{   (VP{[}sem=<{\textbackslash}x.bark(x)>{]} (IV{[}sem=<{\textbackslash}x.bark(x)>{]} barks)))}}\end{alltt}

{\noindent} By default, the semantic representation that is produced by
\texttt{text{\_}interpret{\linebreak[0]}({\linebreak[0]})} has already undergone β-reduction, but in the
above example, we have overridden this.  Subsequent reduction is
possible using the \texttt{simplify{\linebreak[0]}({\linebreak[0]})} method, and Boolean connectives can be
placed in infix position with the \texttt{infixify{\linebreak[0]}({\linebreak[0]})} method.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} semrep.simplify()
\pysrcoutput{exists x.(dog(x) {\&} bark(x))}}\end{alltt}


%___________________________________________________________________________

\subsection{Transitive Verbs}
\label{transitive-verbs}

Our next challenge is to deal with sentences containing transitive
verbs, such as \href{\#sem8}{(172)}.
\hypertarget{sem8}{}\begin{itemize}
\item[(172)] 
Suzie chases a dog.
\end{itemize}

{\noindent} The output semantics that we want to build is shown in \href{\#sem9}{(173)}.
\hypertarget{sem9}{}\begin{itemize}
\item[(173)] 
\texttt{\pysrcstring{'{\linebreak[0]}exists{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\&}{\linebreak[0]} {\linebreak[0]}chase{\linebreak[0]}({\linebreak[0]}suzie{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\end{itemize}

{\noindent} Let's look at how we can use λ-abstraction to get this
result. A significant constraint on possible solutions is to require
that the semantic representation of \docutilsroleexample{a dog} be independent of
whether the \docutilsrolecategory{np} acts as subject or object of the sentence. In
other words, we want to get \href{\#sem9}{(173)} as our output while sticking to
\href{\#sem7}{(170)} as the \docutilsrolecategory{np} semantics. A second constraint is that
\docutilsrolecategory{vp}s should have a uniform type of interpretation regardless
of whether they consist of just an intransitive verb or a transitive
verb plus object. More specifically, we stipulate that \docutilsrolecategory{vp}s
always denote characteristic functions on individuals. Given these
constraints, here's a semantic representation for \docutilsroleexample{chases a dog}
which does the trick.
\hypertarget{sem99}{}\begin{itemize}
\item[(174)] 
\texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}y{\linebreak[0]}.{\linebreak[0]}exists{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\&}{\linebreak[0]} {\linebreak[0]}chase{\linebreak[0]}({\linebreak[0]}y{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\end{itemize}

{\noindent} Think of \href{\#sem99}{(174)} as the property of being a \docutilsrolemath{y} such that
for some dog \docutilsrolemath{x}, \docutilsrolemath{y} chases \docutilsrolemath{x}; or more
colloquially, being a \docutilsrolemath{y} who chases a dog. Our task now
resolves to designing a semantic representation for
\docutilsroleexample{chases} which can combine via \texttt{app} with \href{\#sem7}{(170)} so as to allow
\href{\#sem99}{(174)} to be derived.

Let's carry out a kind of inverse β-reduction on \href{\#sem99}{(174)},
giving rise to \href{\#sem10}{(175)}.

Let Then we are part
way to the solution if we can derive \href{\#sem10}{(175)}, where \texttt{\pysrcstring{'{\linebreak[0]}X{\linebreak[0]}'}} is applied to
\texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}z{\linebreak[0]}.{\linebreak[0]}chase{\linebreak[0]}({\linebreak[0]}y{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}z{\linebreak[0]}){\linebreak[0]}'}}.
\hypertarget{sem10}{}\begin{itemize}
\item[(175)] 
\texttt{\pysrcstring{'{\linebreak[0]}({\linebreak[0]}{\textbackslash}{\linebreak[0]}P{\linebreak[0]}.{\linebreak[0]}exists{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}and{\linebreak[0]} {\linebreak[0]}P{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\textbackslash}{\linebreak[0]}z{\linebreak[0]}.{\linebreak[0]}chase{\linebreak[0]}({\linebreak[0]}y{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}z{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\end{itemize}

{\noindent} \href{\#sem10}{(175)} may be slightly hard to read at first; you need to see that
it involves applying the quantified \docutilsrolecategory{np} representation from
\href{\#sem7}{(170)} to \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}z{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}chase{\linebreak[0]} {\linebreak[0]}z{\linebreak[0]} {\linebreak[0]}y{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}. \href{\#sem10}{(175)} is of course
equivalent to \href{\#sem99}{(174)}.

Now let's replace the functor in \href{\#sem10}{(175)} by a variable \texttt{\pysrcstring{'{\linebreak[0]}X{\linebreak[0]}'}} of the
same type as an \docutilsrolecategory{np}; that is, of type ((\textbf{Ind} → \textbf{Bool})
→ \textbf{Bool}).
\hypertarget{sem11}{}\begin{itemize}
\item[(176)] 
\texttt{\pysrcstring{'{\linebreak[0]}X{\linebreak[0]}({\linebreak[0]}{\textbackslash}{\linebreak[0]}z{\linebreak[0]}.{\linebreak[0]}chase{\linebreak[0]}({\linebreak[0]}y{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}z{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\end{itemize}

{\noindent} The representation of a transitive verb will have to apply to
an argument of the type of \texttt{\pysrcstring{'{\linebreak[0]}X{\linebreak[0]}'}} to yield a functor of the type of
\docutilsrolecategory{vp}s, that is, of type (\textbf{Ind} → \textbf{Bool}). We can ensure
this by abstracting over both the \texttt{\pysrcstring{'{\linebreak[0]}X{\linebreak[0]}'}} variable in \href{\#sem11}{(176)} and also
the subject variable \texttt{\pysrcstring{'{\linebreak[0]}y{\linebreak[0]}'}}. So the full solution is reached by
giving \docutilsroleexample{chases} the semantic representation shown in \href{\#sem12}{(177)}.
\hypertarget{sem12}{}\begin{itemize}
\item[(177)] 
\texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}X{\linebreak[0]} {\linebreak[0]}y{\linebreak[0]}.{\linebreak[0]}X{\linebreak[0]}({\linebreak[0]}{\textbackslash}{\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}chase{\linebreak[0]}({\linebreak[0]}y{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\end{itemize}

{\noindent} If \href{\#sem12}{(177)} is applied to \href{\#sem7}{(170)}, the result after β-reduction is
equivalent to \href{\#sem99}{(174)}, which is what we wanted all along:
\begin{itemize}
\item[(178)] 
\begin{lineblock}{0em}
\item[] \texttt{\pysrcstring{'{\linebreak[0]}({\linebreak[0]}{\textbackslash}{\linebreak[0]}X{\linebreak[0]} {\linebreak[0]}y{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}X{\linebreak[0]} {\linebreak[0]}{\textbackslash}{\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}chase{\linebreak[0]}({\linebreak[0]}y{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\textbackslash}{\linebreak[0]}P{\linebreak[0]}.{\linebreak[0]}exists{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\&}{\linebreak[0]} {\linebreak[0]}P{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\item[] ↝
\item[] \texttt{\pysrcstring{'{\linebreak[0]}({\linebreak[0]}{\textbackslash}{\linebreak[0]}y{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}{\textbackslash}{\linebreak[0]}P{\linebreak[0]}.{\linebreak[0]}exists{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\&}{\linebreak[0]} {\linebreak[0]}P{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\textbackslash}{\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}chase{\linebreak[0]}({\linebreak[0]}y{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\item[] ↝
\item[] \texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}y{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}exists{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\&}{\linebreak[0]} {\linebreak[0]}chase{\linebreak[0]}({\linebreak[0]}y{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\end{lineblock}
\end{itemize}

In order to build a semantic representation for a sentence, we also
need to combine in the semantics of the subject \docutilsrolecategory{np}. If the
latter is a quantified expression like \docutilsroleexample{every girl}, everything
proceeds in the same way as we showed for \docutilsroleexample{a dog barks} earlier
on; the subject is translated as a functor which is applied to the
semantic representation of the \docutilsrolecategory{vp}.  However, we now seem to have
created another problem for ourselves with proper names. So far, these
have been treated semantically as individual constants, and these
cannot be applied as functors to expressions like
\href{\#sem99}{(174)}. Consequently, we need to come up with a different semantic
representation for them. What we do
in this case is re-interpret proper names so that they too are
functors, like quantified \docutilsrolecategory{np}s. \href{\#sem13}{(179)} shows the required
λ expression for \docutilsroleexample{Suzie}.
\hypertarget{sem13}{}\begin{itemize}
\item[(179)] 
\texttt{\pysrcstring{'{\linebreak[0]}{\textbackslash}{\linebreak[0]}P{\linebreak[0]}.{\linebreak[0]}P{\linebreak[0]}({\linebreak[0]}suzie{\linebreak[0]}){\linebreak[0]}'}}
\end{itemize}

{\noindent} \href{\#sem13}{(179)} denotes the characteristic function corresponding to the set of
all properties which are true of Suzie. Converting from an individual
constant to an expression like \href{\#sem12}{(177)} is known as \index{type raising}\textbf{type raising},
and allows us to flip functors with arguments. That is, type raising
means that we can replace a Boolean-valued application such as \docutilsrolemathit{(f
a}) with an equivalent application \docutilsrolemath{(}λ\docutilsrolemathit{P.(P a)
f}).

One important limitation of the approach we have presented here is
that it does not attempt to deal with scope ambiguity. Instead,
quantifier scope ordering directly reflects scope in the parse
tree. As a result, a sentence like \href{\#scope1}{(161)}, repeated here, will always
be translated as \href{\#scope12a}{(181a)}, not \href{\#scope12b}{(181b)}.
\hypertarget{scope11}{}\begin{itemize}
\item[(180)] 
Every girl chases a dog.
\end{itemize}
\hypertarget{scope12}{}\begin{itemize}
\item[(181)] \hypertarget{scope12a}{}\begin{itemize}
\item[a.] 
\texttt{\pysrcstring{'{\linebreak[0]}all{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]}({\linebreak[0]}({\linebreak[0]}girl{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}implies{\linebreak[0]} {\linebreak[0]}some{\linebreak[0]} {\linebreak[0]}y{\linebreak[0]}.{\linebreak[0]} {\linebreak[0]}({\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]} {\linebreak[0]}y{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}and{\linebreak[0]} {\linebreak[0]}({\linebreak[0]}chase{\linebreak[0]} {\linebreak[0]}y{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\end{itemize}
\hypertarget{scope12b}{}\begin{itemize}
\item[b.] 
\texttt{\pysrcstring{'{\linebreak[0]}some{\linebreak[0]} {\linebreak[0]}y{\linebreak[0]}.{\linebreak[0]} {\linebreak[0]}({\linebreak[0]}dog{\linebreak[0]} {\linebreak[0]}y{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}and{\linebreak[0]} {\linebreak[0]}all{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}.{\linebreak[0]} {\linebreak[0]}({\linebreak[0]}({\linebreak[0]}girl{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}implies{\linebreak[0]} {\linebreak[0]}({\linebreak[0]}chase{\linebreak[0]} {\linebreak[0]}y{\linebreak[0]} {\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}){\linebreak[0]}){\linebreak[0]}'}}
\end{itemize}
\end{itemize}

{\noindent} This limitation can be overcome, for example using the hole semantics
described in \cite{blackburn2005rin}, but discussing the details would take
us outside the scope of the current chapter.

Now that we have looked at some slightly more complex constructions, we can
evaluate them in a model. In the following example, we derive two
parses for the sentence \docutilsroleexample{every boy chases a girl in Noosa}, and
evaluate each of the corresponding semantic representations in the
model \texttt{model0{\linebreak[0]}.{\linebreak[0]}py} which we have imported.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }grammar = nltk.data.load(\pysrcstring{'grammars/sem2.fcfg'})
\pysrcprompt{>{}>{}> }val4 = nltk.data.load(\pysrcstring{'grammars/valuation1.val'})
\pysrcprompt{>{}>{}> }dom4 = val4.domain
\pysrcprompt{>{}>{}> }m4 = nltk.sem.Model(dom4, val4)
\pysrcprompt{>{}>{}> }g = nltk.sem.Assignment(dom4)
\pysrcprompt{>{}>{}> }sent = \pysrcstring{'every boy chases a girl in Noosa'}
\pysrcprompt{>{}>{}> }result = nltk.sem.text{\_}evaluate({[}sent{]}, grammar, m4, g)
\pysrcprompt{>{}>{}> }\pysrckeyword{for} (syntree, semrep, value) \pysrckeyword{in} result{[}sent{]}:
\pysrcmore{... }        \pysrckeyword{print} \pysrcstring{"'{\%}s' is {\%}s in Model m{\textbackslash}n"} {\%} (semrep, value)
\pysrcoutput{'all x.(boy(x) -> (exists z2.(girl(z2) {\&} chase(x,z2)) {\&} in(x,noosa)))' is True in Model m}
\pysrcoutput{}
\pysrcoutput{'all x.(boy(x) -> (exists z2.(girl(z2) {\&} chase(x,z2)) {\&} in(x,noosa)))' is True in Model m}
\pysrcoutput{}
\pysrcoutput{'all x.(boy(x) -> exists z3.((girl(z3) {\&} in(z3,noosa)) {\&} chase(x,z3)))' is False in Model m}
\pysrcoutput{}
\pysrcoutput{'all x.(boy(x) -> exists z3.((girl(z3) {\&} in(z3,noosa)) {\&} chase(x,z3)))' is False in Model m}
\pysrcoutput{}}\end{alltt}


%___________________________________________________________________________

\section{Case Study: Extracting Valuations from Chat-80}
\label{case-study-extracting-valuations-from-chat-80}

Building \texttt{Valuation} objects by hand becomes rather tedious once we
consider larger examples. This raises the question of whether the
relation data in a \texttt{Valuation} could be extracted from some
pre-existing source. The \texttt{corpora{\linebreak[0]}.{\linebreak[0]}chat80} module
provides an example of extracting data from the Chat-80 Prolog
knowledge base (which included as part of the NLTK \texttt{corpora}
distribution).

Chat-80 data is organized into collections of clauses, where each
collection functions as a table in a relational database. The
predicate of the clause provides the name of the table; the first
element of the tuple acts as the
'key'; and subsequent elements are further columns in the
table.

In general, the name of the table provides a label for a unary
relation whose extension is all the keys. For example,
the table in \texttt{cities{\linebreak[0]}.{\linebreak[0]}pl} contains triples such as \href{\#city}{(182)}.
\hypertarget{city}{}\begin{itemize}
\item[(182)] 
\texttt{\pysrcstring{'{\linebreak[0]}city{\linebreak[0]}({\linebreak[0]}athens{\linebreak[0]},{\linebreak[0]}greece{\linebreak[0]},{\linebreak[0]}1368{\linebreak[0]}){\linebreak[0]}.{\linebreak[0]}'}}
\end{itemize}

{\noindent} Here, \texttt{\pysrcstring{'{\linebreak[0]}athens{\linebreak[0]}'}} is the key, and will be mapped to a member of the
unary relation \docutilsrolemathit{city}.

The other two columns in the table are mapped to binary relations, where the first
argument of the relation is filled by the table key, and the
second argument is filled by the data in the relevant column. Thus, in
the \texttt{city} table illustrated by the tuple in
\href{\#city}{(182)}, the data from the third column is extracted  into a binary predicate
\docutilsrolemathit{population{\_}of}, whose extension is a set of pairs such as
\texttt{\pysrcstring{'{\linebreak[0]}({\linebreak[0]}athens{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}1368{\linebreak[0]}){\linebreak[0]}'}}.

In order to encapsulate the results of the extraction, a class of
\texttt{Concept}s is introduced.  A \texttt{Concept} object has a number of
attributes, in particular a \texttt{prefLabel} and \texttt{extension}, which
make it easier to inspect the output of the extraction. The
\texttt{extension} of a \texttt{Concept} object is incorporated into a
\texttt{Valuation} object.

As well as deriving unary and binary relations from the Chat-80 data,
we also create a set of individual constants, one for each entity in
the domain. The individual constants are string-identical to the
entities. For example, given a data item such as \texttt{\pysrcstring{'{\linebreak[0]}zloty{\linebreak[0]}'}}, we add
to the valuation a pair \texttt{(\pysrcstring{'{\linebreak[0]}zloty{\linebreak[0]}'},{\linebreak[0]} \pysrcstring{'{\linebreak[0]}zloty{\linebreak[0]}'})}. In order to parse
English sentences that refer to these entities, we also create a
lexical item such as the following for each individual constant:
\begin{itemize}
\item[(183)] \begin{quote}{\ttfamily \raggedright \noindent
\small
PropN{[}num=sg,~sem=<{\textbackslash}P.P(zloty)>{]}~->~'Zloty'
}\end{quote}
\end{itemize}

The \texttt{chat80} module can be found in the \texttt{corpora} package. The
attribute \texttt{chat80{\linebreak[0]}.{\linebreak[0]}items} gives us a list of Chat-80 relations:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk.corpus \pysrckeyword{import} chat80
\pysrcprompt{>{}>{}> }chat80.items 
\pysrcoutput{ ('borders', 'circle{\_}of{\_}lat', 'circle{\_}of{\_}long', 'city', ...)}}\end{alltt}

The \texttt{concepts{\linebreak[0]}({\linebreak[0]})} method shows the list of \texttt{Concept}s that can be
extracted from a \texttt{chat80} relation, and we can then inspect their extensions.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }concepts = chat80.concepts(\pysrcstring{'city'})
\pysrcprompt{>{}>{}> }concepts
\pysrcoutput{{[}Concept('city'), Concept('country{\_}of'), Concept('population{\_}of'){]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }rel = concepts{[}1{]}.extension
\pysrcprompt{>{}>{}> }list(rel){[}:5{]}
\pysrcoutput{{[}('chungking', 'china'), ('karachi', 'pakistan'),}
\pysrcoutput{('singapore{\_}city', 'singapore'), ('athens', 'greece'),}
\pysrcoutput{('birmingham', 'united{\_}kingdom'){]}}}\end{alltt}

{\noindent} In order to convert such an extension into a valuation, we use the
\texttt{make{\_}valuation{\linebreak[0]}({\linebreak[0]})} method; setting \texttt{read{\linebreak[0]}={\linebreak[0]}True} creates and returns
a new \texttt{Valuation} object which contains the results.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }val = nltk.corpus.chat80.make{\_}valuation(concepts, read=True)
\pysrcprompt{>{}>{}> }val{[}\pysrcstring{'city'}{]}{[}\pysrcstring{'calcutta'}{]}
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }val{[}\pysrcstring{'country{\_}of'}{]}{[}\pysrcstring{'india'}{]}
\pysrcoutput{{\{}'hyderabad': True, 'delhi': True, 'bombay': True,}
\pysrcoutput{'madras': True, 'calcutta': True{\}}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }dom = val.domain
\pysrcprompt{>{}>{}> }g = nltk.sem.Assignment(dom)
\pysrcprompt{>{}>{}> }m = nltk.sem.Model(dom, val)
\pysrcprompt{>{}>{}> }m.evaluate(r\pysrcstring{'{\textbackslash}x .population{\_}of(jakarta, x)'}, g)
\pysrcoutput{{\{}'533': True{\}}}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Population figures are given in thousands. Bear in mind that
the geographical data used in these examples dates back at least to
the 1980s, and was already somewhat out of date at the point when
\cite{warren1982eea} was published.
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\section{Summary}
\label{id128}
\begin{itemize}
\item {} 
Semantic Representations (SRs) for English are constructed using a
language based on the λ-calculus, together with Boolean
connectives, equality, and first-order quantifiers.

\item {} 
β-reduction in the λ-calculus corresponds semantically
to application of a function to an argument. Syntactically, it
involves replacing a variable bound by λ in the functor with
the expression that provides the argument in the function
application.

\item {} 
If two λ-abstracts differ only in the label of the variable
bound by λ, they are said to be α
equivalents. Relabeling a variable bound by a λ is called
α-conversion.

\item {} 
Currying of a binary function turns it into a unary function whose
value is again a unary function.

\item {} 
FSRL has both a syntax and a semantics. The semantics is
determined by recursively evaluating expressions in a model.

\item {} 
A key part of constructing a model lies in building a valuation
which assigns interpretations to non-logical constants. These are
interpreted as either curried characteristic functions or as
individual constants.

\item {} 
The interpretation of Boolean connectives is handled by the
model; these are interpreted as characteristic functions.

\item {} 
An open expression is an expression containing one or more free
variables. Open expressions only receive an interpretation when
their free variables receive values from a
variable assignment.

\item {} 
Quantifiers are interpreted by constructing, for a formula φ{[}\docutilsrolemathit{x}{]} open in variable \docutilsrolemathit{x}, the set of individuals
which make φ{[}\docutilsrolemathit{x}{]} true when an assignment \emph{g}
assigns them as the value of \docutilsrolemathit{x}. The quantifier then places
constraints on that set.

\item {} 
A closed expression is one that has no free variables; that is, the
variables are all bound. A closed sentence is true or
false with respect to all variable assignments.

\item {} 
Given a formula with two nested quantifiers \emph{Q}$_{1}$ and \emph{Q}$_{2}$, the outermost quantifier \emph{Q}$_{1}$ is said to have wide
scope (or scope over \emph{Q}$_{2}$). English sentences are frequently
ambiguous with respect to the scope of the quantifiers they
contain.

\item {} 
English sentences can be associated with an SR by treating \texttt{sem}
as a feature. The \texttt{sem} value of a complex expressions typically
involves functional application of the \texttt{sem} values of the
component expressions.

\item {} 
Model valuations need not be built by hand, but can also be
extracted from relational tables, as in the Chat-80 example.

\end{itemize}


%___________________________________________________________________________

\section{Exercises}
\label{id129}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
◑ Modify the \texttt{sem{\linebreak[0]}.{\linebreak[0]}evaluate} code so that it will
give a helpful error message if an expression is not in the domain
of a model's valuation function.

\item {} 
★ Specify and implement a typed functional language with quantifiers,
Boolean connectives and equality. Modify
\texttt{sem{\linebreak[0]}.{\linebreak[0]}evaluate} to interpret expressions of this language.

\item {} 
★ Extend the \texttt{chat80} code so that it will extract data from a
relational database using SQL queries.

\item {} 
★ Taking {[}WarrenPereira1982{]} as a starting point, develop a technique
for converting a natural language query into a form that can be
evaluated more efficiently in a model. For example, given a query
of the form \texttt{\pysrcstring{'{\linebreak[0]}({\linebreak[0]}P{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\&}{\linebreak[0]} {\linebreak[0]}Q{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}'}}, convert it to \texttt{\pysrcstring{'{\linebreak[0]}({\linebreak[0]}Q{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]} {\linebreak[0]}{\&}{\linebreak[0]} {\linebreak[0]}P{\linebreak[0]}({\linebreak[0]}x{\linebreak[0]}){\linebreak[0]}'}} if
the extension of \texttt{\pysrcstring{'{\linebreak[0]}Q{\linebreak[0]}'}} is smaller than the extension of
\texttt{\pysrcstring{'{\linebreak[0]}P{\linebreak[0]}'}}.

\end{list}


%___________________________________________________________________________

\section{Further Reading}
\label{id130}

For more examples of semantic analysis with NLTK, please see the
guides at
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}sem{\linebreak[0]}.{\linebreak[0]}html} and
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides{\linebreak[0]}/{\linebreak[0]}logic{\linebreak[0]}.{\linebreak[0]}html}.

The use of characteristic functions for interpreting expressions of
natural language was primarily due to Richard
Montague. \cite{dowty1981ims} gives a comprehensive and reasonably
approachable introduction to Montague's grammatical framework.

A more recent and wide-reaching study of the use of a λ based
approach to natural language can be found in \cite{carpenter1997tls}.

\cite{heim1998sgg} is a thorough application of formal semantics to
transformational grammars in the Government-Binding model.

\cite{blackburn2005rin} is the first textbook devoted to computational
semantics, and provides an excellent introduction to the area.
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
\hypertarget{chap-data}{}\setcounter{chapter}{11}

%___________________________________________________________________________

\chapter{Linguistic Data Management (DRAFT)}
\label{linguistic-data-management-draft}
\label{chap-data}
% TODO: random syllables according to a canon "%s%s" % (random.choice('ptkbdg'), random.choice('aeiou')) 
% TODO: linguistic annotation 
% TODO: paradigms 
% TODO: exercise on generating comparative wordlists 


%___________________________________________________________________________

\section{Introduction}
\label{id135}

Language resources of all kinds are proliferating on the Web.  These include data such as
lexicons and annotated text, and software tools for creating and manipulating the data.
As we have seen in previous chapters, language resources are essential in most areas of NLP.
This has been made possible by three significant technological developments over the past decade.
First, inexpensive mass storage technology permits large resources to be stored in
digital form, while the Extensible Markup Language (XML) and Unicode provide flexible ways
to represent structured data and give it good prospects for long-term survival.
Second, digital publication has been a practical and efficient means of sharing language
resources.  Finally, search engines, mailing lists, and online resource catalogs
make it possible for people to discover the existence of the resources they may be seeking.

Together with these technological advances have been three other developments that have
shifted the NLP community in the direction of data-intensive approaches.  First, the
''shared task method,`` an initiative of government sponsors, supports
major sections within the community to identify a common goal for the coming year, and
provides ''gold standard`` data on which competing systems can be evaluated.
Second, data publishers such as the Linguistic Data Consortium have
negotiated with hundreds of data providers (including newswire services in many countries),
and created hundreds of annotated corpora stored in well-defined and consistent formats.
Finally, organizations that purchase NLP systems, or that publish NLP papers,
now expect the quality of the work to be demonstrated using standard datasets.

Although language resources are central to NLP, we still face many obstacles in using them.
First, the resource we are looking for may not exist, and so we have to think about
\emph{creating} a new language resource, and doing a sufficiently careful job that it serves
our future needs, thanks to its coverage, balance, and documentation of the sources.
Second, a resource may exist but its creator didn't document its existence anywhere,
leaving us to recreate the resource; however, to save further wasted effort we
should learn about \emph{publishing} metadata the documents the existence of a resource,
and even how to publish the resource itself, in a form that is easy for others to re-use.
Third, the resource may exist and may be obtained, but is in an incompatible format,
and so we need to set about \emph{converting} the data into a different format.
Finally, the resource may be in the right format, but the available software is unable
to perform the required analysis task, and so we need to develop our own program
for \emph{analyzing} the data.  This chapter covers each of these issues
— creating, publishing, converting, and analyzing — using
many examples drawn from practical experience managing linguistic data.
However, before embarking on this sequence of issues, we start by examining the
organization of linguistic data.
\hypertarget{sec-organization-linguistic-data}{}

%___________________________________________________________________________

\section{Linguistic Databases}
\label{linguistic-databases}
\label{sec-organization-linguistic-data}

Linguistic databases span a multidimensional space of cases, which we
can divide up in several ways: the scope and design of the data
collection; the goals of the creators; the nature of the material
included; the goals and methods of the users (which are often not
anticipated by the creators). Three examples follow.

In one type of linguistic database, the design unfolds interactively
in the course of the creator's explorations. This is the pattern
typical of traditional ''field linguistics,`` in which material from
elicitation sessions is analyzed
repeatedly as it is gathered, with tomorrow's elicitation often based
on questions that arise in analyzing today's. The resulting field
notes are then used during subsequent years of research, and may serve
as an archival resource indefinitely — the field notes of linguists
and anthropologists working in the early years of the 20th century
remain an important source of information today. Computerization is an
obvious boon to work of this type, as exemplified by the popular
program \index{Shoebox}\textbf{Shoebox} — now about two decades old and re-released as \index{Toolbox}\textbf{Toolbox} —
which replaces the field linguist's traditional shoebox full of file cards.

Another pattern is represented by experimental approaches in which a
body of carefully-designed material is collected from a range of subjects,
then analyzed to evaluate a hypothesis or develop a technology.
Today, such databases are collected and analyzed in digital
form.  Among scientists (such as phoneticians or psychologists), they
are rarely published and therefore rarely preserved. Among engineers,
it has become common for such databases to be shared and re-used at
least within a laboratory or company, and often to be published more
widely. Linguistic databases of this type are the basis of the
''common task`` method of research management, which over the
past 15 years has become the norm in government-funded research programs
in speech- and language-related technology.

Finally, there are efforts to gather a ''reference corpus`` for a
particular language. Large and well-documented examples include
the \index{American National Corpus}\textbf{American National Corpus} (ANC) and the
\index{British National Corpus}\textbf{British National Corpus} (BNC). The goal in
such cases is to produce a set of linguistic materials that cover the
many forms, styles and uses of a language as widely as possible. The
core application is typically lexicographic, that is, the construction
of dictionaries based on a careful study of patterns of use. These
corpora were constructed by large consortia spanning government, industry,
and academia.  Their planning and execution took more than five years, and
indirectly involved hundreds of person-years of effort. There is also a long
and distinguished history of other humanistic reference corpora, such
the Thesaurus Linguae Graecae.

There are no hard boundaries among these categories. Accumulations of
smaller bodies of data may come in time to constitute a sort of reference
corpus, while selections from large databases may form the basis for
a particular experiment. Further instructive examples follow.

A linguist's field notes may include extensive examples of many genres
(proverbs, conversations, narratives, rituals, and so forth), and may
come to constitute a reference corpus of modest but useful size. There
are many extinct languages for which such material is all the data we
will ever have, and many more endangered languages for which such
documentation is urgently needed. Sociolinguists typically base their
work on analysis of a set of recorded interviews, which may over time
grow to create another sort of reference corpus. In some labs, the
residue of decades of work may comprise literally thousands of hours
of recordings, many of which have been transcribed and annotated to
one extent or another.  The \index{CHILDES}\textbf{CHILDES} corpus, comprising
transcriptions of parent-child interactions in many languages,
contributed by many individual researchers, has come to constitute a
widely-used reference corpus for language acquisition
research. Speech technologists aim to produce training and testing
material of broad applicability, and wind up creating
another sort of reference corpus.  To date, linguistic technology
R{\&}D has been the primary source of published linguistic databases of
all sorts (see e.g. \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}www{\linebreak[0]}.{\linebreak[0]}ldc{\linebreak[0]}.{\linebreak[0]}upenn{\linebreak[0]}.{\linebreak[0]}edu{\linebreak[0]}/}).

As large, varied linguistic databases are published, phoneticians or
psychologists are increasingly likely to base experimental
investigations on balanced, focused subsets extracted from databases
produced for entirely different reasons. Their motivations include the
desire to save time and effort, the desire to work on material
available to others for replication, and sometimes a desire to study
more naturalistic forms of linguistic behavior. The process of
choosing a subset for such a study, and making the measurements
involved, is usually in itself a non-trivial addition to the
database. This recycling of linguistic databases for new purposes is a
normal and expected consequence of publication.  For instance, the
Switchboard database, originally collected for speaker identification research,
has since been used as the basis for published studies in speech recognition, word pronunciation,
disfluency, syntax, intonation and discourse structure.

At present, only a tiny fraction of the linguistic databases that are
collected are published in any meaningful sense. This is mostly
because publication of such material was both time-consuming and
expensive, and because use of such material by other researchers was
also both expensive and technically difficult. However, general
improvements in hardware, software and networking have changed this,
and linguistic databases can now be created, published,
stored and used without inordinate effort or large expense.

In practice, the implications of these cost-performance changes are
only beginning to be felt.  The main problem is that adequate tools
for creation, publication and use of linguistic data are not widely
available. In most cases, each project must create its own set of
tools, which hinders publication by researchers who lack the
expertise, time or resources to make their data accessible to others.
Furthermore, we do not have adequate, generally accepted standards for
expressing the structure and content of linguistic databases. Without
such standards, general-purpose tools are impossible — though at the
same time, without available tools, adequate standards are unlikely to
be developed, used and accepted.  Just as importantly, there must be a
critical mass of users and published material to motivate maintenance
of data and access tools over time.

Relative to these needs, the present chapter has modest goals, namely to
equip readers to take linguistic databases into their own hands by writing
programs to help create, publish, transform and analyze the data.  In the
rest of this section we take a close look at the fundamental data types,
an exemplary speech corpus, and the lifecycle of linguistic data.
\hypertarget{datatypes}{}\begin{figure}[htbp]
\centering

\scalebox{0.180000}{\includegraphics{../images/datatypes.png}}
\caption{Basic Linguistic Datatypes: Lexicons and Texts}\end{figure}
\hypertarget{sec-fundamental-data-types}{}

%___________________________________________________________________________

\subsection{Fundamental Data Types}
\label{fundamental-data-types}
\label{sec-fundamental-data-types}

Linguistic data management deals with a variety of data types, the most
important being lexicons and texts. A \index{lexicon}\textbf{lexicon} is a database
of words, minimally containing part of speech information and
glosses.  For many lexical resources, it is sufficient to use a
\index{record}\textbf{record} structure, i.e. a key plus one or more fields, as
shown in \href{\#datatypes}{Figure 12.1}.  A lexical resource could be a conventional
dictionary or comparative wordlist, as illustrated.
Several related linguistic data types also fit this model.  For example
in a phrasal lexicon, the key field is a phrase rather than a single word.
A thesaurus can be derived from a lexicon by adding topic fields to the
entries and constructing an index over those fields.
We can also construct special tabulations (known as paradigms)
to illustrate contrasts and systematic variation, as shown in \href{\#datatypes}{Figure 12.1} for three verbs.

At the most abstract level, a \index{text}\textbf{text} is a representation of a real or fictional speech event,
and the time-course of that event carries over into the text itself.  A text could be a small
unit, such as a word or sentence, or a complete narrative or dialogue.  It may come with
annotations such as part-of-speech tags, morphological analysis, discourse structure, and so forth.
As we saw in the IOB tagging technique (\href{\#chap-chunk}{Chapter 6}), it is possible to represent higher-level
constituents using tags on individual words.  Thus the abstraction of text shown in
\href{\#datatypes}{Figure 12.1} is sufficient.


%___________________________________________________________________________

\subsection{Corpus Structure: a Case Study of TIMIT}
\label{corpus-structure-a-case-study-of-timit}

The TIMIT corpus of read speech was the first annotated speech database to be
widely distributed, and it has an especially clear organization.
TIMIT was developed by a consortium including Texas Instruments and MIT (hence the name),
and was designed to provide data for the acquisition of acoustic-phonetic knowledge and to
support the development and evaluation of automatic speech recognition systems.

Like the Brown Corpus, which displays a balanced selection of text genres and sources,
TIMIT includes a balanced selection of dialects, speakers, and materials.  For each of
eight dialect regions, 50 male and female speakers having a range of ages and educational
backgrounds each read ten carefully chosen sentences.  Two sentences, read by all
speakers, were designed to bring out dialect variation:
\begin{itemize}
\item[(184)] \begin{itemize}
\item[a.] 
she had your dark suit in greasy wash water all year
\end{itemize}
\begin{itemize}
\item[b.] 
don't ask me to carry an oily rag like that
\end{itemize}
\end{itemize}

The remaining sentences were chosen to be phonetically rich, involving all phones (sounds) and
a comprehensive range of diphones (phone bigrams).  Additionally, the design strikes a balance
between multiple speakers saying the same sentence in order to permit comparison across
speakers, and having a large range of sentences covered by the corpus to get maximal
coverage of diphones.  Thus, five sentences read by each speaker, are also read by six other
speakers (comparability).  The remaining three sentences read by each speaker were unique to that speaker (coverage).

NLTK includes a sample from the TIMIT corpus.  You can access its documentation in the usual
way, using \texttt{help{\linebreak[0]}({\linebreak[0]}corpus{\linebreak[0]}.{\linebreak[0]}timit{\linebreak[0]})}.  Print \texttt{corpus{\linebreak[0]}.{\linebreak[0]}timit{\linebreak[0]}.{\linebreak[0]}items} to see a list of the
160 recorded utterances in the corpus sample.
Each item name has complex internal structure, as shown in \href{\#timit}{Figure 12.2}.
\hypertarget{timit}{}\begin{figure}[htbp]
\centering

\scalebox{0.154286}{\includegraphics{../images/timit.png}}
\caption{Structure of a TIMIT Identifier in the NLTK Corpus Package}\end{figure}

Each item has a phonetic transcription, which can be accessed using the \texttt{phones{\linebreak[0]}({\linebreak[0]})}
method.  We can access the corresponding word tokens in the customary way.  Both access
methods permit an optional argument \texttt{offset{\linebreak[0]}={\linebreak[0]}True} which includes the start and end offsets
of the corresponding span in the audio file.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }phonetic = nltk.corpus.timit.phones(\pysrcstring{'dr1-fvmh0/sa1'})
\pysrcprompt{>{}>{}> }phonetic
\pysrcoutput{{[}'h{\#}', 'sh', 'iy', 'hv', 'ae', 'dcl', 'y', 'ix', 'dcl', 'd', 'aa', 'kcl',}
\pysrcoutput{'s', 'ux', 'tcl', 'en', 'gcl', 'g', 'r', 'iy', 's', 'iy', 'w', 'aa',}
\pysrcoutput{'sh', 'epi', 'w', 'aa', 'dx', 'ax', 'q', 'ao', 'l', 'y', 'ih', 'ax', 'h{\#}'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.corpus.timit.word{\_}times(\pysrcstring{'dr1-fvmh0/sa1'})
\pysrcoutput{{[}('she', 7812, 10610), ('had', 10610, 14496), ('your', 14496, 15791),}
\pysrcoutput{('dark', 15791, 20720), ('suit', 20720, 25647), ('in', 25647, 26906),}
\pysrcoutput{('greasy', 26906, 32668), ('wash', 32668, 37890), ('water', 38531, 42417),}
\pysrcoutput{('all', 43091, 46052), ('year', 46052, 50522){]}}}\end{alltt}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Remember that our program samples assume you
begin your interactive session or your program with: \texttt{\pysrckeyword{import} {\linebreak[0]}nltk{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}re{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}pprint}
(Note that some of the examples in this chapter have not yet been updated to
work with NLTK-Lite version 0.9).
\end{boxedminipage}
\end{sffamily}
\end{center}

In addition to this text data, TIMIT includes a lexicon that provides the canonical
pronunciation of every word:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }timitdict = nltk.corpus.timit.transcription{\_}dict()
\pysrcprompt{>{}>{}> }timitdict{[}\pysrcstring{'greasy'}{]} + timitdict{[}\pysrcstring{'wash'}{]} + timitdict{[}\pysrcstring{'water'}{]}
\pysrcoutput{{[}'g', 'r', 'iy1', 's', 'iy', 'w', 'ao1', 'sh', 'w', 'ao1', 't', 'axr'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }phonetic{[}17:30{]}
\pysrcoutput{{[}'g', 'r', 'iy', 's', 'iy', 'w', 'aa', 'sh', 'epi', 'w', 'aa', 'dx', 'ax'{]}}}\end{alltt}

{\noindent}
This gives us a sense of what a speech processing system
would have to do in producing or recognizing speech in this particular dialect
(New England).  Finally, TIMIT includes demographic data about the speakers,
permitting fine-grained study of vocal, social, and gender characteristics.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nltk.corpus.timit.spkrinfo(\pysrcstring{'dr1-fvmh0'})
\pysrcoutput{SpeakerInfo(id='VMH0', sex='F', dr='1', use='TRN', recdate='03/11/86',}
\pysrcoutput{birthdate='01/08/60', ht='5{\textbackslash}'05"', race='WHT', edu='BS',}
\pysrcoutput{comments='BEST NEW ENGLAND ACCENT SO FAR')}}\end{alltt}

TIMIT illustrates several key features of corpus design.
First, the corpus contains two layers of annotation, at the phonetic and orthographic
levels.  In general, a text or speech corpus may be annotated at many different linguistic levels,
including morphological, syntactic, and discourse levels.  Moreover, even at a given
level there may be different labeling schemes or even disagreement amongst annotators,
such that we want to represent multiple versions.
A second property of TIMIT is its balance across multiple dimensions of variation,
for coverage of dialect regions and diphones.  The inclusion of speaker
demographics brings in many more independent variables, that may help to
account for variation in the data, and which facilitate later uses of the
corpus for purposes that were not envisaged when the corpus was created,
e.g. sociolinguistics.
A third property is that there is a sharp division between the original
linguistic event captured as an audio recording, and the annotations of that event.
The same holds true of text corpora, in the sense that the original text usually
has an external source, and is considered to be an immutable artifact.  Any transformations
of that artifact which involve human judgment — even something as
simple as tokenization — are subject to later revision, thus it is important to
retain the source material in a form that is as close to the original as possible.
\hypertarget{timit-structure}{}\begin{figure}[htbp]
\centering

\scalebox{0.154286}{\includegraphics{../images/timit-structure.png}}
\caption{Structure of the Published TIMIT Corpus}\end{figure}

A fourth feature of TIMIT is the hierarchical structure of the corpus.
With 4 files per sentence, and 10 sentences for each of 500 speakers, there are 20,000 files.  These are
organized into a tree structure, shown schematically in \href{\#timit-structure}{Figure 12.3}.
At the top level there is a split between training and testing sets, which gives
away its intended use for developing and evaluating statistical models.

Finally, notice that even though TIMIT is a speech corpus, its transcriptions and associated
data are just text, and can be processed using programs just like any other text corpus.
Therefore, many of the computational methods described in this book are applicable.
Moreover, notice that all of the data types included in the TIMIT corpus fall into
our two basic categories of lexicon and text (cf. \href{\#sec-fundamental-data-types}{section 12.2.1}).
Even the speaker demographics data is just another instance of the lexicon data type.

This last observation is less surprising when we consider that text and record structures are the
primary domains for the two subfields of computer science that focus on data management,
namely text retrieval and databases.  A notable feature of linguistic data management is
that usually brings both data types together, and that it can draw on results and techniques
from both fields.


%___________________________________________________________________________

\subsection{The Lifecycle of Linguistic Data: Evolution vs Curation}
\label{the-lifecycle-of-linguistic-data-evolution-vs-curation}

Once a corpus has been created and disseminated, it typically gains a life of its own,
as others adapt it to their needs.  This may involve reformatting a text file (e.g. converting
to XML), renaming files, retokenizing the text, selecting a subset of the data to enrich,
and so forth.  Multiple research groups may do this work independently, as exemplified
in \href{\#evolution}{Figure 12.4}.  At a later date, when someone wants to combine sources
of information from different version, the task may be extremely onerous.
\hypertarget{evolution}{}\begin{figure}[htbp]
\centering

\scalebox{0.185143}{\includegraphics{../images/evolution.png}}
\caption{Evolution of a Corpus}\end{figure}

The task of using derived corpora is made even more difficult by the lack of
any record about how the derived version was created, and which version is the most
up-to-date.

An alternative to this chaotic situation is for all corpora to be centrally curated,
and for committees of experts to revise and extend a reference corpus at periodic
intervals, considering proposals for new content from third-parties, much like a dictionary is edited.
However, this is impractical.

A better solution is to have a canonical, immutable primary source, which supports
incoming references to any sub-part, and then for all annotations (including segmentations)
to reference this source.  This way, two independent tokenizations of the same text
can be represented without touch the source text, as can any further labeling and grouping
of those annotations.  This method is known as \index{standoff annotation}\textbf{standoff annotation}.

{[}More discussion and examples{]}


%___________________________________________________________________________

\section{Creating Data}
\label{creating-data}

Scenarios: fieldwork, web, manual entry using local tool, machine learning with manual post-editing

Conventional office software is widely used in computer-based language
documentation work, given its familiarity and ready availability.
This includes word processors and spreadsheets.


%___________________________________________________________________________

\subsection{Spiders}
\label{spiders}
\begin{itemize}
\item {} 
what they do: basic idea is simple

\item {} 
python code to find all the anchors, extract the href, and make an absolute URL for fetching

\item {} 
many issues: starting points, staying within a single site, only getting HTML

\item {} 
various stand-alone tools for spidering, and mirroring

\end{itemize}


%___________________________________________________________________________

\subsection{Creating Language Resources Using Word Processors}
\label{creating-language-resources-using-word-processors}

Word processing software is often used in creating dictionaries and
interlinear texts.  As the data grows in size and complexity, a larger
proportion of time is spent maintaining consistency.
Consider a dictionary in
which each entry has a part-of-speech field, drawn from a set of 20
possibilities, displayed after the pronunciation field, and rendered
in 11-point bold.  No conventional word processor has search or macro
functions capable of verifying that all part-of-speech fields have
been correctly entered and displayed.  This task requires exhaustive
manual checking.  If the word processor permits the document to be
saved in a non-proprietary format, such as text, HTML, or XML, we can
sometimes write programs to do this checking automatically.

Consider the following fragment of a lexical entry:
''sleep {[}sli:p{]} \textbf{vi} \emph{condition of body and mind...}``.
We can enter this in MSWord, then ''Save as Web Page``,
then inspect the resulting HTML file:
\begin{quote}{\ttfamily \raggedright \noindent
\small
<p~class=MsoNormal>sleep~\\
~~<span~style='mso-spacerun:yes'>~</span>~\\
~~{[}<span~class=SpellE>sli:p</span>{]}~\\
~~<span~style='mso-spacerun:yes'>~</span>~\\
~~<b><span~style='font-size:11.0pt'>vi</span></b>~\\
~~<span~style='mso-spacerun:yes'>~</span>~\\
~~<i>a~condition~of~body~and~mind~...<o:p></o:p></i>~\\
</p>
}\end{quote}

Observe that the entry is represented as an HTML paragraph, using the
\texttt{<{\linebreak[0]}p{\linebreak[0]}>} element, and that the part of speech appears inside a \texttt{<{\linebreak[0]}span
style{\linebreak[0]}=\pysrcstring{'{\linebreak[0]}font{\linebreak[0]}-{\linebreak[0]}size{\linebreak[0]}:{\linebreak[0]}11{\linebreak[0]}.{\linebreak[0]}0pt{\linebreak[0]}'}>} element.  The following program defines
the set of legal parts-of-speech, \texttt{legal{\_}pos}.  Then it extracts all
11-point content from the \texttt{dict{\linebreak[0]}.{\linebreak[0]}htm} file and stores it in the set
\texttt{used{\_}pos}.  Observe that the search pattern contains a
parenthesized sub-expression; only the material that matches this
sub-expression is returned by \texttt{re{\linebreak[0]}.{\linebreak[0]}findall}.  Finally, the program
constructs the set of illegal parts-of-speech as \texttt{used{\_}pos{\linebreak[0]} {\linebreak[0]}-
legal{\_}pos}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }legal{\_}pos = set({[}\pysrcstring{'n'}, \pysrcstring{'v.t.'}, \pysrcstring{'v.i.'}, \pysrcstring{'adj'}, \pysrcstring{'det'}{]})
\pysrcprompt{>{}>{}> }pattern = re.compile(r\pysrcstring{"'font-size:11.0pt'>({[}a-z.{]}+)<"})
\pysrcprompt{>{}>{}> }document = open(\pysrcstring{"dict.htm"}).read()
\pysrcprompt{>{}>{}> }used{\_}pos = set(re.findall(pattern, document))
\pysrcprompt{>{}>{}> }illegal{\_}pos = used{\_}pos.difference(legal{\_}pos)
\pysrcprompt{>{}>{}> }\pysrckeyword{print} list(illegal{\_}pos)
\pysrcoutput{{[}'v.i', 'intrans'{]}}}\end{alltt}

This simple program represents the tip of the iceberg.  We can develop
sophisticated tools to check the consistency of word processor files,
and report errors so that the maintainer of the dictionary can correct
the original file \emph{using the original word processor}.

We can write other programs to convert the data into a different format.
For example, \href{\#html2csv}{Listing 12.1} strips out the HTML markup using \texttt{ntlk{\linebreak[0]}.{\linebreak[0]}clean{\_}html{\linebreak[0]}({\linebreak[0]})},
extracts the words and their pronunciations, and generates output
in ''comma-separated value`` (CSV) format:
\hypertarget{html2csv}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{lexical{\_}data}(html{\_}file):
    SEP = \pysrcstring{'{\_}ENTRY'}
    html = open(html{\_}file).read()
    html = re.sub(r\pysrcstring{'<p'}, SEP + \pysrcstring{'<p'}, html)
    text = nltk.clean{\_}html(html)
    text = \pysrcstring{' '}.join(text.split())
    \pysrckeyword{for} entry \pysrckeyword{in} text.split(SEP):
        \pysrckeyword{if} entry.count(\pysrcstring{' '}) > 2:
            yield entry.split(\pysrcstring{' '}, 3)}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} csv
\pysrcprompt{>{}>{}> }writer = csv.writer(open(\pysrcstring{"dict1.csv"}, \pysrcstring{"wb"}))
\pysrcprompt{>{}>{}> }writer.writerows(lexical{\_}data(\pysrcstring{"dict.htm"}))}\end{alltt}
\caption{Converting HTML Created by Microsoft Word into Comma-Separated Values}
\vspace{1ex}
\hrule
\end{pylisting}


%___________________________________________________________________________

\subsection{Creating Language Resources Using Spreadsheets and Databases}
\label{creating-language-resources-using-spreadsheets-and-databases}

\textbf{Spreadsheets.} These are often used for wordlists or paradigms.  A
comparative wordlist may be stored in a spreadsheet, with a row for
each cognate set, and a column for each language.  Examples are
available from \texttt{www{\linebreak[0]}.{\linebreak[0]}rosettaproject{\linebreak[0]}.{\linebreak[0]}org}.  Programs such as Excel can
export spreadsheets in the CSV format, and we can write programs to
manipulate them, with the help of Python's \texttt{csv} module.  For
example, we may want to print out cognates having an edit-distance of
at least three from each other (i.e. 3 insertions, deletions, or
substitutions).

\textbf{Databases.} Sometimes lexicons are stored in a full-fledged
relational database.  When properly normalized, these databases can
implement many well-formedness constraints.  For example, we can
require that all parts-of-speech come from a specified vocabulary by
declaring that the part-of-speech field is an \emph{enumerated type}.
However, the relational model is often too restrictive for linguistic
data, which typically has many optional and repeatable fields (e.g.
dictionary sense definitions and example sentences).
Query languages such as SQL cannot express many
linguistically-motivated queries, e.g. \emph{find all words that appear
in example sentences for which no dictionary entry is provided}.
Now supposing that the database supports exporting data to CSV format,
and that we can save the data to a file \texttt{dict{\linebreak[0]}.{\linebreak[0]}csv}:
\begin{quote}{\ttfamily \raggedright \noindent
\small
"sleep","sli:p","v.i","a~condition~of~body~and~mind~..."~\\
"walk","wo:k","v.intr","progress~by~lifting~and~setting~down~each~foot~..."~\\
"wake","weik","intrans","cease~to~sleep"
}\end{quote}

Now we can express this query as shown in \href{\#undefined-words}{Figure 12.2}.
\hypertarget{undefined-words}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{def} \pysrcdefname{undefined{\_}words}(csv{\_}file):
    \pysrckeyword{import} csv
    lexemes = set()
    defn{\_}words = set()
    \pysrckeyword{for} row \pysrckeyword{in} csv.reader(open(csv{\_}file)):
        lexeme, pron, pos, defn = row
        lexemes.add(lexeme)
        defn{\_}words.union(defn.split())
    return sorted(defn{\_}words.difference(lexemes))}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{print} undefined{\_}words(\pysrcstring{"dict.csv"})
\pysrcoutput{{[}'...', 'a', 'and', 'body', 'by', 'cease', 'condition', 'down', 'each',}
\pysrcoutput{'foot', 'lifting', 'mind', 'of', 'progress', 'setting', 'to'{]}}}\end{alltt}
\caption{Finding definition words not themselves defined}
\vspace{1ex}
\hrule
\end{pylisting}


%___________________________________________________________________________

\subsection{Creating Language Resources Using Toolbox}
\label{creating-language-resources-using-toolbox}

Over the last two decades, several dozen tools have been developed
that provide specialized support for linguistic data management.
Perhaps the single most popular tool used by linguists for managing data
is \emph{Toolbox}, previously known as \emph{Shoebox}
(freely downloadable from \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}www{\linebreak[0]}.{\linebreak[0]}sil{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}computing{\linebreak[0]}/{\linebreak[0]}toolbox{\linebreak[0]}/}).
In this section we discuss a variety of
techniques for manipulating Toolbox data in ways that are not supported
by the Toolbox software.  (The methods we discuss could be
applied to other record-structured data, regardless of the actual file format.)

A Toolbox file consists of a collection of \emph{entries} (or \emph{records}),
where each record is made up of one or more \emph{fields}.  Here is an
example of an entry taken from a Toolbox dictionary of Rotokas.
(Rotokas is an East Papuan language spoken on the island of
Bougainville; this data was provided by Stuart Robinson, and
is a sample from a larger lexicon):
\begin{quote}{\ttfamily \raggedright \noindent
\small
{\textbackslash}lx~kaa~\\
{\textbackslash}ps~N~\\
{\textbackslash}pt~MASC~\\
{\textbackslash}cl~isi~\\
{\textbackslash}ge~cooking~banana~\\
{\textbackslash}tkp~banana~bilong~kukim~\\
{\textbackslash}pt~itoo~\\
{\textbackslash}sf~FLORA~\\
{\textbackslash}dt~12/Aug/2005~\\
{\textbackslash}ex~Taeavi~iria~kaa~isi~kovopaueva~kaparapasia.~\\
{\textbackslash}xp~Taeavi~i~bin~planim~gaden~banana~bilong~kukim~tasol~long~paia.~\\
{\textbackslash}xe~Taeavi~planted~banana~in~order~to~cook~it.
}\end{quote}
% what does the other pt mean? 

This lexical entry contains the following fields:
\texttt{lx} lexeme;
\texttt{ps} part-of-speech;
\texttt{pt} part-of-speech;
\texttt{cl} classifier;
\texttt{ge} English gloss;
\texttt{tkp} Tok Pisin gloss;
\texttt{sf} Semantic field;
\texttt{dt} Date last edited;
\texttt{ex} Example sentence;
\texttt{xp} Pidgin translation of example;
\texttt{xe} English translation of example.
These field names are preceded by a backslash, and must always appear
at the start of a line.  The characters of the field names must be
alphabetic.  The field name is separated from the field's contents by
whitespace.  The contents can be arbitrary text, and can continue over
several lines (but cannot contain a line-initial backslash).

We can use the \texttt{toolbox{\linebreak[0]}.{\linebreak[0]}xml{\linebreak[0]}({\linebreak[0]})} method to access a Toolbox
file and load it into an \texttt{elementtree} object.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk.corpus \pysrckeyword{import} toolbox
\pysrcprompt{>{}>{}> }lexicon = toolbox.xml(\pysrcstring{'rotokas.dic'})}\end{alltt}

There are two ways to access the contents of the lexicon object, by
indexes and by paths.  Indexes use the familiar syntax, thus
\texttt{lexicon{\linebreak[0]}{[}{\linebreak[0]}3{\linebreak[0]}{]}} returns entry number 3 (which is actually the fourth
entry counting from zero).  And \texttt{lexicon{\linebreak[0]}{[}{\linebreak[0]}3{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}{]}} returns its first
field:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }lexicon{[}3{]}{[}0{]}
\pysrcoutput{<Element lx at 77bd28>}
\pysrcoutput{}\pysrcprompt{>{}>{}> }lexicon{[}3{]}{[}0{]}.tag
\pysrcoutput{'lx'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }lexicon{[}3{]}{[}0{]}.text
\pysrcoutput{'kaa'}}\end{alltt}

The second way to access the contents of the lexicon object uses
paths.  The lexicon is a series of \texttt{record} objects, each containing
a series of field objects, such as \texttt{lx} and \texttt{ps}.  We can
conveniently address all of the lexemes using the path \texttt{record{\linebreak[0]}/{\linebreak[0]}lx}.
Here we use the \texttt{findall{\linebreak[0]}({\linebreak[0]})} function to search for any matches to
the path \texttt{record{\linebreak[0]}/{\linebreak[0]}lx}, and we access the text content of the element,
normalizing it to lowercase.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }{[}lexeme.text.lower() \pysrckeyword{for} lexeme \pysrckeyword{in} lexicon.findall(\pysrcstring{'record/lx'}){]}
\pysrcoutput{{[}'kaa', 'kaa', 'kaa', 'kaakaaro', 'kaakaaviko', 'kaakaavo', 'kaakaoko',}
\pysrcoutput{'kaakasi', 'kaakau', 'kaakauko', 'kaakito', 'kaakuupato', ..., 'kuvuto'{]}}}\end{alltt}

It is often convenient to add new fields that are derived automatically from
existing ones.  Such fields often facilitate search and analysis.
For example, in \href{\#add-cv-field}{Listing 12.3} we define a function \texttt{cv{\linebreak[0]}({\linebreak[0]})} which
maps a string of consonants and vowels to the corresponding CV sequence,
e.g. \texttt{kakapua} would map to \texttt{CVCVCVV}.
This mapping has four steps.  First, the string is converted to lowercase,
then we replace any non-alphabetic characters \texttt{{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}} with an underscore.
Next, we replace all vowels with \texttt{V}.  Finally, anything that is not
a \texttt{V} or an underscore must be a consonant, so we replace it with a \texttt{C}.
Now, we can scan the lexicon and add a new \texttt{cv} field after every \texttt{lx} field.
\href{\#add-cv-field}{Listing 12.3} shows what this does to a particular entry; note
the last line of output, which shows the new CV field.
\hypertarget{add-cv-field}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{from} nltk.etree.ElementTree \pysrckeyword{import} SubElement

\pysrckeyword{def} \pysrcdefname{cv}(s):
    s = s.lower()
    s = re.sub(r\pysrcstring{'{[}{\textasciicircum}a-z{]}'},     r\pysrcstring{'{\_}'}, s)
    s = re.sub(r\pysrcstring{'{[}aeiou{]}'},    r\pysrcstring{'V'}, s)
    s = re.sub(r\pysrcstring{'{[}{\textasciicircum}V{\_}{]}'},      r\pysrcstring{'C'}, s)
    return (s)

\pysrckeyword{def} \pysrcdefname{add{\_}cv{\_}field}(entry):
    \pysrckeyword{for} field \pysrckeyword{in} entry:
        \pysrckeyword{if} field.tag == \pysrcstring{'lx'}:
            cv{\_}field = SubElement(entry, \pysrcstring{'cv'})
            cv{\_}field.text = cv(field.text)}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }lexicon = toolbox.xml(\pysrcstring{'rotokas.dic'})
\pysrcprompt{>{}>{}> }add{\_}cv{\_}field(lexicon{[}53{]})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} nltk.corpus.reader.toolbox.to{\_}sfm{\_}string(lexicon{[}53{]})
\pysrcoutput{{\textbackslash}lx kaeviro}
\pysrcoutput{{\textbackslash}ps V}
\pysrcoutput{{\textbackslash}pt A}
\pysrcoutput{{\textbackslash}ge lift off}
\pysrcoutput{{\textbackslash}ge take off}
\pysrcoutput{{\textbackslash}tkp go antap}
\pysrcoutput{{\textbackslash}sc MOTION}
\pysrcoutput{{\textbackslash}vx 1}
\pysrcoutput{{\textbackslash}nt used to describe action of plane}
\pysrcoutput{{\textbackslash}dt 03/Jun/2005}
\pysrcoutput{{\textbackslash}ex Pita kaeviroroe kepa kekesia oa vuripierevo kiuvu.}
\pysrcoutput{{\textbackslash}xp Pita i go antap na lukim haus win i bagarapim.}
\pysrcoutput{{\textbackslash}xe Peter went to look at the house that the wind destroyed.}
\pysrcoutput{{\textbackslash}cv CVVCVCV}}\end{alltt}
\caption{Adding a new cv field to a lexical entry}
\vspace{1ex}
\hrule
\end{pylisting}

Finally, we take a look at simple methods to generate summary reports,
giving us an overall picture of the quality and organisation of the data.

First, suppose that we wanted to compute the average number of fields for
each entry.  This is just the total length of the entries (the number
of fields they contain), divided by the number of entries in the
lexicon:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }sum(len(entry) \pysrckeyword{for} entry \pysrckeyword{in} lexicon) / len(lexicon)
\pysrcoutput{13}}\end{alltt}
\hypertarget{toolbox-validation}{}
\begin{pylisting}
\begin{alltt}\small\textbf{grammar = nltk.parse{\_}cfg(\pysrcstring{'{}'{}'}
\pysrcstring{  S -> Head PS Glosses Comment Date Examples}
\pysrcstring{  Head -> Lexeme Root}
\pysrcstring{  Lexeme -> "lx"}
\pysrcstring{  Root -> "rt" |}
\pysrcstring{  PS -> "ps"}
\pysrcstring{  Glosses -> Gloss Glosses |}
\pysrcstring{  Gloss -> "ge" | "gp"}
\pysrcstring{  Date -> "dt"}
\pysrcstring{  Examples -> Example Ex{\_}Pidgin Ex{\_}English Examples |}
\pysrcstring{  Example -> "ex"}
\pysrcstring{  Ex{\_}Pidgin -> "xp"}
\pysrcstring{  Ex{\_}English -> "xe"}
\pysrcstring{  Comment -> "cmt" |}
\pysrcstring{  '{}'{}'})

\pysrckeyword{def} \pysrcdefname{validate{\_}lexicon}(grammar, lexicon):
    rd{\_}parser = nltk.RecursiveDescentParser(grammar)
    \pysrckeyword{for} entry \pysrckeyword{in} lexicon{[}10:20{]}:
        marker{\_}list = {[}field.tag \pysrckeyword{for} field \pysrckeyword{in} entry{]}
        \pysrckeyword{if} rd{\_}parser.get{\_}parse{\_}list(marker{\_}list):
            \pysrckeyword{print} \pysrcstring{"+"}, \pysrcstring{':'}.join(marker{\_}list)
        \pysrckeyword{else}:
            \pysrckeyword{print} \pysrcstring{"-"}, \pysrcstring{':'}.join(marker{\_}list)}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }lexicon = toolbox.xml(\pysrcstring{'rotokas.dic'}){[}10:20{]}
\pysrcprompt{>{}>{}> }validate{\_}lexicon(grammar, lexicon)
\pysrcoutput{- lx:ps:ge:gp:sf:nt:dt:ex:xp:xe:ex:xp:xe}
\pysrcoutput{- lx:rt:ps:ge:gp:nt:dt:ex:xp:xe:ex:xp:xe}
\pysrcoutput{- lx:ps:ge:gp:nt:dt:ex:xp:xe:ex:xp:xe}
\pysrcoutput{- lx:ps:ge:gp:nt:sf:dt}
\pysrcoutput{- lx:ps:ge:gp:dt:cmt:ex:xp:xe:ex:xp:xe}
\pysrcoutput{+ lx:ps:ge:ge:ge:gp:cmt:dt:ex:xp:xe}
\pysrcoutput{+ lx:rt:ps:ge:gp:cmt:dt:ex:xp:xe:ex:xp:xe}
\pysrcoutput{+ lx:rt:ps:ge:ge:gp:dt}
\pysrcoutput{- lx:rt:ps:ge:ge:ge:gp:dt:cmt:ex:xp:xe:ex:xp:xe:ex:xp:xe}
\pysrcoutput{+ lx:rt:ps:ge:gp:dt:ex:xp:xe}}\end{alltt}
\caption{Validating Toolbox Entries Using a Context Free Grammar}
\vspace{1ex}
\hrule
\end{pylisting}

We could try to write
down a grammar for lexical entries, and look for entries which do not
conform to the grammar.  In general, toolbox entries have nested
structure.  Thus they correspond to a tree over the fields.  We can
check for well-formedness by parsing the field names.
In \href{\#chunk-toolbox}{Listing 12.5} we set up
a putative grammar for the entries, then parse each entry.
Those that are accepted by the grammar prefixed with
a \texttt{\pysrcstring{'{\linebreak[0]}+{\linebreak[0]}'}}, and those that are rejected are prefixed with a \texttt{\pysrcstring{'{\linebreak[0]}-{\linebreak[0]}'}}.
\hypertarget{chunk-toolbox}{}
\begin{pylisting}
\begin{alltt}\small\textbf{\pysrckeyword{import} os.path, sys
\pysrckeyword{from} nltk{\_}contrib \pysrckeyword{import} toolbox

grammar = r\pysrcstring{"{}"{}"}
\pysrcstring{      lexfunc: {\{}<lf>(<lv><ln|le>*)*{\}}}
\pysrcstring{      example: {\{}<rf|xv><xn|xe>*{\}}}
\pysrcstring{      sense:   {\{}<sn><ps><pn|gv|dv|gn|gp|dn|rn|ge|de|re>*<example>*<lexfunc>*{\}}}
\pysrcstring{      record:   {\{}<lx><hm><sense>+<dt>{\}}}
\pysrcstring{    "{}"{}"}}\end{alltt}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk.etree.ElementTree \pysrckeyword{import} ElementTree
\pysrcprompt{>{}>{}> }db = toolbox.ToolboxData()
\pysrcprompt{>{}>{}> }db.open(nltk.data.find(\pysrcstring{'corpora/toolbox/iu{\_}mien{\_}samp.db'}))
\pysrcprompt{>{}>{}> }lexicon = db.chunk{\_}parse(grammar, encoding=\pysrcstring{'utf8'})
\pysrcprompt{>{}>{}> }toolbox.data.indent(lexicon)
\pysrcprompt{>{}>{}> }tree = ElementTree(lexicon)
\pysrcprompt{>{}>{}> }tree.write(sys.stdout, encoding=\pysrcstring{'utf8'})}\end{alltt}
\caption{Chunking a Toolbox Lexicon}
\vspace{1ex}
\hrule
\end{pylisting}


%___________________________________________________________________________

\subsection{Interlinear Text}
\label{interlinear-text}

The NLTK corpus collection includes many interlinear text samples
(though no suitable corpus reader as yet).

\index{General Ontology for Linguistic Description}\textbf{General Ontology for Linguistic Description} (GOLD)
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}www{\linebreak[0]}.{\linebreak[0]}linguistics{\linebreak[0]}-{\linebreak[0]}ontology{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/}


%___________________________________________________________________________

\subsection{Creating Metadata for Language Resources}
\label{creating-metadata-for-language-resources}

OLAC metadata extends the \index{Dublin Core}\textbf{Dublin Core} metadata set with descriptors
that are important for language resources.

The container for an OLAC metadata record is the element \texttt{<{\linebreak[0]}olac{\linebreak[0]}>}.
Here is a valid OLAC metadata record from the Pacific And Regional
Archive for Digital Sources in Endangered Cultures (PARADISEC):
\begin{quote}{\ttfamily \raggedright \noindent
\small
<olac:olac~xsi:schemaLocation="http://purl.org/dc/elements/1.1/~http://www.language-archives.org/OLAC/1.0/dc.xsd~\\
~~http://purl.org/dc/terms/~http://www.language-archives.org/OLAC/1.0/dcterms.xsd~\\
~~http://www.language-archives.org/OLAC/1.0/~http://www.language-archives.org/OLAC/1.0/olac.xsd">~\\
~~<dc:title>Tiraq~Field~Tape~019</dc:title>~\\
~~<dc:identifier>AB1-019</dc:identifier>~\\
~~<dcterms:hasPart>AB1-019-A.mp3</dcterms:hasPart>~\\
~~<dcterms:hasPart>AB1-019-A.wav</dcterms:hasPart>~\\
~~<dcterms:hasPart>AB1-019-B.mp3</dcterms:hasPart>~\\
~~<dcterms:hasPart>AB1-019-B.wav</dcterms:hasPart>~\\
~~<dc:contributor~xsi:type="olac:role"~olac:code="recorder">Brotchie,~Amanda</dc:contributor>~\\
~~<dc:subject~xsi:type="olac:language"~olac:code="x-sil-MME"/>~\\
~~<dc:language~xsi:type="olac:language"~olac:code="x-sil-BCY"/>~\\
~~<dc:language~xsi:type="olac:language"~olac:code="x-sil-MME"/>~\\
~~<dc:format>Digitised:~yes;</dc:format>~\\
~~<dc:type>primary{\_}text</dc:type>~\\
~~<dcterms:accessRights>standard,~as~per~PDSC~Access~form</dcterms:accessRights>~\\
~~<dc:description>SIDE~A<p>1.~Elicitation~Session~-~Discussion~and~\\
~~~~translation~of~Lise's~and~Marie-Claire's~Songs~and~Stories~from~\\
~~~~Tape~18~(Tamedal)<p><p>SIDE~B<p>1.~Elicitation~Session:~Discussion~\\
~~~~of~and~translation~of~Lise's~and~Marie-Clare's~songs~and~stories~\\
~~~~from~Tape~018~(Tamedal)<p>2.~Kastom~Story~1~-~Bislama~\\
~~~~(Alec).~Language~as~given:~Tiraq</dc:description>~\\
</olac:olac>
}\end{quote}

NLTK Version 0.9 includes support for reading an OLAC record, for example:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }file = nltk.data.find(\pysrcstring{'corpora/treebank/olac.xml'})
\pysrcprompt{>{}>{}> }xml = open(file).read()
\pysrcprompt{>{}>{}> }nltk.olac.pprint{\_}olac(xml)
\pysrcoutput{identifier  : LDC99T42}
\pysrcoutput{title       : Treebank-3}
\pysrcoutput{type        : (olac:linguistic-type=primary{\_}text)}
\pysrcoutput{description : Release type: General}
\pysrcoutput{creator     : Mitchell P. Marcus, Beatrice Santorini, Mary Ann Marcinkiewicz and Ann Taylor (olac:role=author)}
\pysrcoutput{identifier  : ISBN: 1-58563-163-9}
\pysrcoutput{description : Online documentation: http://www.ldc.upenn.edu/Catalog/docs/treebank3/}
\pysrcoutput{subject     : English (olac:language=x-sil-ENG)}}\end{alltt}


%___________________________________________________________________________

\subsection{Linguistic Annotation}
\label{linguistic-annotation}

Annotation graph model

multiple overlapping trees over shared data

Large annotation tasks require multiple annotators.  How consistently can a group of annotators
perform?  It is insufficient to report that there is 80{\%} agreement, as we have no way to tell if
this is good or bad.  I.e. for an easy task such as tagging, this would be a bad score, while for
a difficult task such as semantic role labeling, this would be an exceptionally good score.

The \index{Kappa}\textbf{Kappa} coefficient K measures agreement between two people making category
judgments, correcting for expected chance agreement.  For example, suppose an item is to
be annotated, and four coding options are equally likely.  Then people coding randomly would
be expected to agree 25{\%} of the time.  Thus, an agreement of 25{\%} will be assigned K = 0,
and better levels of agreement will be scaled accordingly.  For an agreement of 50{\%}, we would
get K = 0.333, as 50 is a third of the way from 25 to 100.


%___________________________________________________________________________

\subsection{Exercises}
\label{id136}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
☼ Write a program to filter out just the date field (\texttt{dt}) without
having to list the fields we wanted to retain.

\item {} 
☼ Print an index of a lexicon.  For each lexical entry, construct a
tuple of the form \texttt{({\linebreak[0]}gloss{\linebreak[0]},{\linebreak[0]} {\linebreak[0]}lexeme{\linebreak[0]})}, then sort and print them all.

\item {} 
☼ What is the frequency of each consonant and vowel contained in
lexeme fields?

\item {} 
◑ In \href{\#add-cv-field}{Listing 12.3} the new field appeared at the bottom of the
entry.  Modify this program so that it inserts the new subelement right after
the \texttt{lx} field.  (Hint: create the new \texttt{cv} field using \texttt{Element{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}cv{\linebreak[0]}'})},
assign a text value to it, then use the \texttt{insert{\linebreak[0]}({\linebreak[0]})} method of the parent element.)

\item {} 
◑ Write a function that deletes a specified field from a lexical entry.
(We could use this to sanitize our lexical data before giving it to others,
e.g. by removing fields containing irrelevant or uncertain content.)

\item {} 
◑ Write a program that scans an HTML dictionary file to find entries
having an illegal part-of-speech field, and reports the \emph{headword} for
each entry.

\item {} 
◑ Write a program to find any parts of speech (\texttt{ps} field) that
occurred less than ten times.  Perhaps these are typing mistakes?

\item {} 
◑ We saw a method for discovering cases of whole-word reduplication.
Write a function to find words that may contain partial
reduplication.  Use the \texttt{re{\linebreak[0]}.{\linebreak[0]}search{\linebreak[0]}({\linebreak[0]})} method, and the following
regular expression: \texttt{({\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}+{\linebreak[0]}){\linebreak[0]}{\textbackslash}{\linebreak[0]}1}

\item {} 
◑ We saw a method for adding a \texttt{cv} field.  There is an interesting
issue with keeping this up-to-date when someone modifies the content
of the \texttt{lx} field on which it is based.  Write a version of this
program to add a \texttt{cv} field, replacing any existing \texttt{cv} field.

\item {} 
◑ Write a function to add a new field \texttt{syl} which gives a count of
the number of syllables in the word.

\item {} 
◑ Write a function which displays the complete entry for a lexeme.
When the lexeme is incorrectly spelled it should display the entry
for the most similarly spelled lexeme.

\item {} 
◑ Write a function that takes a lexicon and finds which pairs of
consecutive fields are most frequent (e.g. \texttt{ps} is often followed by \texttt{pt}).
(This might help us to discover some of the structure of a lexical entry.)

\item {} 
★ Obtain a comparative wordlist in CSV format, and write a program
that prints those cognates having an edit-distance of at least three
from each other.

\item {} 
★ Build an index of those lexemes which appear in example sentences.
Suppose the lexeme for a given entry is \emph{w}.
Then add a single cross-reference field \texttt{xrf} to this entry, referencing
the headwords of other entries having example sentences containing
\emph{w}.  Do this for all entries and save the result as a toolbox-format file.

\end{list}


%___________________________________________________________________________

\section{Converting Data Formats}
\label{converting-data-formats}
\begin{itemize}
\item {} 
write our own parser and formatted print

\item {} 
use existing libraries, e.g. csv

\end{itemize}


%___________________________________________________________________________

\subsection{Formatting Entries}
\label{formatting-entries}

We can also print a formatted
version of a lexicon.  It allows us to request specific fields without
needing to be concerned with their relative ordering in the original file.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }lexicon = toolbox.xml(\pysrcstring{'rotokas.dic'})
\pysrcprompt{>{}>{}> }\pysrckeyword{for} entry \pysrckeyword{in} lexicon{[}70:80{]}:
\pysrcmore{... }    lx = entry.findtext(\pysrcstring{'lx'})
\pysrcmore{... }    ps = entry.findtext(\pysrcstring{'ps'})
\pysrcmore{... }    ge = entry.findtext(\pysrcstring{'ge'})
\pysrcmore{... }    \pysrckeyword{print} \pysrcstring{"{\%}s ({\%}s) '{\%}s'"} {\%} (lx, ps, ge)
\pysrcoutput{kakae (???) 'small'}
\pysrcoutput{kakae (CLASS) 'child'}
\pysrcoutput{kakaevira (ADV) 'small-like'}
\pysrcoutput{kakapikoa (???) 'small'}
\pysrcoutput{kakapikoto (N) 'newborn baby'}
\pysrcoutput{kakapu (V) 'place in sling for purpose of carrying'}
\pysrcoutput{kakapua (N) 'sling for lifting'}
\pysrcoutput{kakara (N) 'arm band'}
\pysrcoutput{Kakarapaia (N) 'village name'}
\pysrcoutput{kakarau (N) 'frog'}}\end{alltt}

We can use the same idea to generate HTML tables instead of plain text.
This would be useful for publishing a Toolbox lexicon on the web.
It produces HTML elements \texttt{<{\linebreak[0]}table{\linebreak[0]}>}, \texttt{<{\linebreak[0]}tr{\linebreak[0]}>} (table row), and
\texttt{<{\linebreak[0]}td{\linebreak[0]}>} (table data).
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }html = \pysrcstring{"<table>{\textbackslash}n"}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} entry \pysrckeyword{in} lexicon{[}70:80{]}:
\pysrcmore{... }    lx = entry.findtext(\pysrcstring{'lx'})
\pysrcmore{... }    ps = entry.findtext(\pysrcstring{'ps'})
\pysrcmore{... }    ge = entry.findtext(\pysrcstring{'ge'})
\pysrcmore{... }    html += \pysrcstring{"  <tr><td>{\%}s</td><td>{\%}s</td><td>{\%}s</td></tr>{\textbackslash}n"} {\%} (lx, ps, ge)
\pysrcprompt{>{}>{}> }html += \pysrcstring{"</table>"}
\pysrcprompt{>{}>{}> }\pysrckeyword{print} html
\pysrcoutput{<table>}
\pysrcoutput{  <tr><td>kakae</td><td>???</td><td>small</td></tr>}
\pysrcoutput{  <tr><td>kakae</td><td>CLASS</td><td>child</td></tr>}
\pysrcoutput{  <tr><td>kakaevira</td><td>ADV</td><td>small-like</td></tr>}
\pysrcoutput{  <tr><td>kakapikoa</td><td>???</td><td>small</td></tr>}
\pysrcoutput{  <tr><td>kakapikoto</td><td>N</td><td>newborn baby</td></tr>}
\pysrcoutput{  <tr><td>kakapu</td><td>V</td><td>place in sling for purpose of carrying</td></tr>}
\pysrcoutput{  <tr><td>kakapua</td><td>N</td><td>sling for lifting</td></tr>}
\pysrcoutput{  <tr><td>kakara</td><td>N</td><td>arm band</td></tr>}
\pysrcoutput{  <tr><td>Kakarapaia</td><td>N</td><td>village name</td></tr>}
\pysrcoutput{  <tr><td>kakarau</td><td>N</td><td>frog</td></tr>}
\pysrcoutput{</table>}}\end{alltt}

XML output
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} sys
\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk.etree.ElementTree \pysrckeyword{import} ElementTree
\pysrcprompt{>{}>{}> }tree = ElementTree(lexicon{[}3{]})
\pysrcprompt{>{}>{}> }tree.write(sys.stdout)
\pysrcoutput{<record>}
\pysrcoutput{  <lx>kaa</lx>}
\pysrcoutput{  <ps>N</ps>}
\pysrcoutput{  <pt>MASC</pt>}
\pysrcoutput{  <cl>isi</cl>}
\pysrcoutput{  <ge>cooking banana</ge>}
\pysrcoutput{  <tkp>banana bilong kukim</tkp>}
\pysrcoutput{  <pt>itoo</pt>}
\pysrcoutput{  <sf>FLORA</sf>}
\pysrcoutput{  <dt>12/Aug/2005</dt>}
\pysrcoutput{  <ex>Taeavi iria kaa isi kovopaueva kaparapasia.</ex>}
\pysrcoutput{  <xp>Taeavi i bin planim gaden banana bilong kukim tasol long paia.</xp>}
\pysrcoutput{  <xe>Taeavi planted banana in order to cook it.</xe>}
\pysrcoutput{</record>}}\end{alltt}


%___________________________________________________________________________

\subsection{Exercises}
\label{id137}
\begin{itemize}
\item {} 
◑ Create a spreadsheet using office software, containing one lexical entry per
row, consisting of a headword, a part of speech, and a gloss.  Save the
spreadsheet in CSV format.  Write Python code to read the CSV file and print it in
Toolbox format, using \texttt{lx} for the headword, \texttt{ps} for the part of speech,
and \texttt{gl} for the gloss.

\end{itemize}


%___________________________________________________________________________

\section{Analyzing Language Data}
\label{analyzing-language-data}

I.e. linguistic exploration

Export to statistics package via CSV

In this section we consider a variety of analysis tasks.

\textbf{Reduplication:}
First, we will develop a program to find reduplicated words.  In order
to do this we need to store all verbs, along with their English glosses.
We need to keep the glosses so that they can be displayed alongside the
wordforms.  The following code defines a Python dictionary \texttt{lexgloss}
which maps verbs to their English glosses:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }lexgloss = {\{}{\}}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} entry \pysrckeyword{in} lexicon:
\pysrcmore{... }    lx = entry.findtext(\pysrcstring{'lx'})
\pysrcmore{... }    ps = entry.findtext(\pysrcstring{'ps'})
\pysrcmore{... }    \pysrckeyword{if} lx \pysrckeyword{and} ps \pysrckeyword{and} ps{[}0{]} == \pysrcstring{'V'}:
\pysrcmore{... }        lexgloss{[}lx{]} = entry.findtext(\pysrcstring{'ge'})
\pysrcoutput{kasi (burn); kasikasi (angry)}
\pysrcoutput{kee (shatter); keekee (chipped)}
\pysrcoutput{kauo (jump); kauokauo (jump up and down)}
\pysrcoutput{kea (confused); keakea (lie)}
\pysrcoutput{kape (unable to meet); kapekape (embrace)}
\pysrcoutput{kapo (fasten.cover.strip); kapokapo (fasten.cover.strips)}
\pysrcoutput{kavo (collect); kavokavo (perform sorcery)}
\pysrcoutput{karu (open); karukaru (open)}
\pysrcoutput{kare (return); karekare (return)}
\pysrcoutput{kari (rip); karikari (tear)}
\pysrcoutput{kae (blow); kaekae (tempt)}}\end{alltt}

Next, for each verb \texttt{lex}, we will check if the lexicon contains the
reduplicated form \texttt{lex{\linebreak[0]}+{\linebreak[0]}lex}.  If it does, we report both forms along with
their glosses.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} lex \pysrckeyword{in} lexgloss:
\pysrcmore{... }    \pysrckeyword{if} lex+lex \pysrckeyword{in} lexgloss:
\pysrcmore{... }        \pysrckeyword{print} \pysrcstring{"{\%}s ({\%}s); {\%}s ({\%}s)"} {\%}{\textbackslash}
\pysrcmore{... }            (lex, lexgloss{[}lex{]}, lex+lex, lexgloss{[}lex+lex{]})
\pysrcoutput{kuvu (fill.up); kuvukuvu (fill up)}
\pysrcoutput{kitu (store); kitukitu (scrub clothes)}
\pysrcoutput{kiru (have sore near mouth); kirukiru (crisp)}
\pysrcoutput{kopa (swallow); kopakopa (gulp.down)}
\pysrcoutput{kasi (burn); kasikasi (angry)}
\pysrcoutput{koi (high pitched sound); koikoi (groan with pain)}
\pysrcoutput{kee (shatter); keekee (chipped)}
\pysrcoutput{kauo (jump); kauokauo (jump up and down)}
\pysrcoutput{kea (confused); keakea (lie)}
\pysrcoutput{kovo (work); kovokovo (play)}
\pysrcoutput{kove (fell); kovekove (drip repeatedly)}
\pysrcoutput{kape (unable to meet); kapekape (embrace)}
\pysrcoutput{kapo (fasten.cover.strip); kapokapo (fasten.cover.strips)}
\pysrcoutput{koa (skin); koakoa (bark a tree)}
\pysrcoutput{kipu (paint); kipukipu (rub.on)}
\pysrcoutput{koe (spoon out a solid); koekoe (spoon out)}
\pysrcoutput{kotu (bite); kotukotu (gnash teeth)}
\pysrcoutput{kavo (collect); kavokavo (perform sorcery)}
\pysrcoutput{kuri (scrape); kurikuri (scratch repeatedly)}
\pysrcoutput{karu (open); karukaru (open)}
\pysrcoutput{kare (return); karekare (return)}
\pysrcoutput{kari (rip); karikari (tear)}
\pysrcoutput{kiro (write); kirokiro (write)}
\pysrcoutput{kae (blow); kaekae (tempt)}
\pysrcoutput{koru (make return); korukoru (block)}
\pysrcoutput{kosi (exit); kosikosi (exit)}}\end{alltt}

\textbf{Complex Search Criteria:}
Phonological description typically identifies the segments, alternations,
syllable canon and so forth.  It is relatively
straightforward to count up the occurrences of all the different types
of CV syllables that occur in lexemes.

In the following example, we first import the regular expression and
probability modules.  Then we iterate over the lexemes to find all
sequences of a non-vowel \texttt{{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]}aeiou{\linebreak[0]}{]}} followed by a vowel \texttt{{[}{\linebreak[0]}aeiou{\linebreak[0]}{]}}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }fd = nltk.FreqDist()
\pysrcprompt{>{}>{}> }tokenizer = nltk.RegexpTokenizer(pattern=r\pysrcstring{'{[}{\textasciicircum}aeiou{]}{[}aeiou{]}'})
\pysrcprompt{>{}>{}> }lexemes = {[}lexeme.text.lower() \pysrckeyword{for} lexeme \pysrckeyword{in} lexicon.findall(\pysrcstring{'record/lx'}){]}
\pysrcprompt{>{}>{}> }\pysrckeyword{for} lex \pysrckeyword{in} lexemes:
\pysrcmore{... }    \pysrckeyword{for} syl \pysrckeyword{in} tokenizer.tokenize(lex):
\pysrcmore{... }        fd.inc(syl)}\end{alltt}

Now, rather than just printing the syllables and their frequency
counts, we can tabulate them to generate a useful display.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} vowel \pysrckeyword{in} \pysrcstring{'aeiou'}:
\pysrcmore{... }    \pysrckeyword{for} cons \pysrckeyword{in} \pysrcstring{'ptkvsr'}:
\pysrcmore{... }         \pysrckeyword{print} \pysrcstring{'{\%}s{\%}s:{\%}4d '} {\%} (cons, vowel, fd{[}cons+vowel{]}),
\pysrcmore{... }    \pysrckeyword{print}
\pysrcoutput{pa:  83  ta:  47  ka: 428  va:  93  sa:   0  ra: 187}
\pysrcoutput{pe:  31  te:   8  ke: 151  ve:  27  se:   0  re:  63}
\pysrcoutput{pi: 105  ti:   0  ki:  94  vi: 105  si: 100  ri:  84}
\pysrcoutput{po:  34  to: 148  ko: 430  vo:  48  so:   2  ro:  89}
\pysrcoutput{pu:  51  tu:  37  ku: 175  vu:  49  su:   1  ru:  79}}\end{alltt}

Consider the \texttt{t} and \texttt{s} columns, and observe that \texttt{ti} is not
attested, while \texttt{si} is frequent.  This suggests that a phonological
process of palatalization is operating in the language.  We would then
want to consider the other syllables involving \texttt{s}
(e.g. the single entry having \emph{su}, namely \emph{kasuari} 'cassowary' is a loanword).

\textbf{Prosodically-motivated search:}
A phonological description may include an examination of the segmental
and prosodic constraints on well-formed morphemes and lexemes.  For
example, we may want to find trisyllabic verbs ending in a long vowel.
Our program can make use of the fact that syllable onsets are
obligatory and simple (only consist of a single consonant).
First, we will encapsulate the syllabic counting part in a separate
function.  It gets the CV template of the word \texttt{cv{\linebreak[0]}({\linebreak[0]}word{\linebreak[0]})} and counts
the number of consonants it contains:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{num{\_}cons}(word):
\pysrcmore{... }    template = cv(word)
\pysrcmore{... }    return template.count(\pysrcstring{'C'})}\end{alltt}

We also encapsulate the vowel test in a function, as this improves the
readability of the final program.  This function returns the value
\texttt{True} just in case \texttt{char} is a vowel.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{def} \pysrcdefname{is{\_}vowel}(char):
\pysrcmore{... }    return (char \pysrckeyword{in} \pysrcstring{'aeiou'})}\end{alltt}

Over time we may create a useful collection of such functions.  We can
save them in a file \texttt{utilities{\linebreak[0]}.{\linebreak[0]}py}, and then at the start of each
program we can simply import all the functions in one go using \texttt{\pysrckeyword{from}
utilities{\linebreak[0]} \pysrckeyword{import} {\linebreak[0]}*}.  We take the entry to be a verb if the first
letter of its part of speech is a \texttt{V}.  Here, then, is the program
to display trisyllabic verbs ending in a long vowel:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} entry \pysrckeyword{in} lexicon:
\pysrcmore{... }    lx = entry.findtext(\pysrcstring{'lx'})
\pysrcmore{... }    \pysrckeyword{if} lx:
\pysrcmore{... }        ps = entry.findtext(\pysrcstring{'ps'})
\pysrcmore{... }        \pysrckeyword{if} num{\_}cons(lx) == 3 \pysrckeyword{and} ps{[}0{]} == \pysrcstring{'V'}{\textbackslash}
\pysrcmore{... }          \pysrckeyword{and} is{\_}vowel(lx{[}-1{]}) \pysrckeyword{and} is{\_}vowel(lx{[}-2{]}):
\pysrcmore{... }            ge = entry.findtext(\pysrcstring{'ge'})
\pysrcmore{... }            \pysrckeyword{print} \pysrcstring{"{\%}s ({\%}s) '{\%}s'"} {\%} (lx, ps, ge)
\pysrcoutput{kaetupie (V) 'tighten'}
\pysrcoutput{kakupie (V) 'shout'}
\pysrcoutput{kapatau (V) 'add to'}
\pysrcoutput{kapuapie (V) 'wound'}
\pysrcoutput{kapupie (V) 'close tight'}
\pysrcoutput{kapuupie (V) 'close'}
\pysrcoutput{karepie (V) 'return'}
\pysrcoutput{karivai (V) 'have an appetite'}
\pysrcoutput{kasipie (V) 'care for'}
\pysrcoutput{kasirao (V) 'hot'}
\pysrcoutput{kaukaupie (V) 'shine intensely'}
\pysrcoutput{kavorou (V) 'covet'}
\pysrcoutput{kavupie (V) 'leave.behind'}
\pysrcoutput{kekepie (V) 'show'}
\pysrcoutput{keruria (V) 'persistent'}
\pysrcoutput{ketoopie (V) 'make sprout from seed'}
\pysrcoutput{kipapie (V) 'wan samting tru'}
\pysrcoutput{koatapie (V) 'put in'}
\pysrcoutput{koetapie (V) 'investigate'}
\pysrcoutput{koikoipie (V) 'make groan with pain'}
\pysrcoutput{kokepie (V) 'make.rain'}
\pysrcoutput{kokoruu (V) 'insect-infested'}
\pysrcoutput{kokovae (V) 'sing'}
\pysrcoutput{kokovua (V) 'shave the hair line'}
\pysrcoutput{kopiipie (V) 'kill'}
\pysrcoutput{korupie (V) 'take outside'}
\pysrcoutput{kosipie (V) 'make exit'}
\pysrcoutput{kovopie (V) 'make work'}
\pysrcoutput{kukuvai (V) 'shelter head'}
\pysrcoutput{kuvaupie (V) 'desert'}}\end{alltt}

\textbf{Finding Minimal Sets:}
In order to establish a contrast segments (or lexical properties, for
that matter), we would like to find pairs of words which are identical
except for a single property.  For example, the words pairs \emph{mace} vs
\emph{maze} and \emph{face} vs \emph{faze} — and many others like them — demonstrate the
existence of a phonemic distinction between \emph{s} and \emph{z} in English.
NLTK provides flexible support for constructing minimal sets, using
the \texttt{MinimalSet{\linebreak[0]}({\linebreak[0]})} class.  This class needs three pieces of information
for each item to be added:
\texttt{context}: the material that must be fixed across all members of a minimal set;
\texttt{target}: the material that changes across members of a minimal set;
\texttt{display}: the material that should be displayed for each item.

\leavevmode
\begin{longtable}[c]{|p{0.307\locallinewidth}|p{0.226\locallinewidth}|p{0.191\locallinewidth}|p{0.145\locallinewidth}|}
\hline
\multicolumn{4}{|l|}{\textbf{
Examples of Minimal Set Parameters
}} \\
\hline
\textbf{
Minimal Set
} & \textbf{
Context
} & \textbf{
Target
} & \textbf{
Display
} \\
\hline
\endhead

\emph{bib}, \emph{bid}, \emph{big}
 & 
first two letters
 & 
third letter
 & 
word
 \\
\hline

\emph{deal (N)}, \emph{deal (V)}
 & 
whole word
 & 
pos
 & 
word (pos)
 \\
\hline
\caption{}\end{longtable}

We begin by creating a list of parameter values, generated from the
full lexical entries.  In our first example, we will print minimal sets
involving lexemes of length 4, with a target position of 1 (second
segment).  The \texttt{context} is taken to be the entire word, except for
the target segment.  Thus, if lex is \texttt{kasi}, then context is
\texttt{lex{\linebreak[0]}{[}{\linebreak[0]}:{\linebreak[0]}1{\linebreak[0]}{]}{\linebreak[0]}+\pysrcstring{'{\linebreak[0]}{\_}{\linebreak[0]}'}+{\linebreak[0]}lex{\linebreak[0]}{[}{\linebreak[0]}2{\linebreak[0]}:{\linebreak[0]}{]}}, or \texttt{k{\_}si}.  Note that no parameters are
generated if the lexeme does not consist of exactly four segments.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }pos = 1
\pysrcprompt{>{}>{}> }ms = nltk.MinimalSet((lex{[}:pos{]} + \pysrcstring{'{\_}'} + lex{[}pos+1:{]}, lex{[}pos{]}, lex)
\pysrcmore{... }                      \pysrckeyword{for} lex \pysrckeyword{in} lexemes \pysrckeyword{if} len(lex) == 4)}\end{alltt}

Now we print the table of minimal sets.  We specify that each
context was seen at least 3 times.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{for} context \pysrckeyword{in} ms.contexts(3):
\pysrcmore{... }    \pysrckeyword{print} context + \pysrcstring{':'},
\pysrcmore{... }    \pysrckeyword{for} target \pysrckeyword{in} ms.targets():
\pysrcmore{... }        \pysrckeyword{print} \pysrcstring{"{\%}-4s"} {\%} ms.display(context, target, \pysrcstring{"-"}),
\pysrcmore{... }    \pysrckeyword{print}
\pysrcoutput{k{\_}si: kasi -    kesi kusi kosi}
\pysrcoutput{k{\_}va: kava -    -    kuva kova}
\pysrcoutput{k{\_}ru: karu kiru keru kuru koru}
\pysrcoutput{k{\_}pu: kapu kipu -    -    kopu}
\pysrcoutput{k{\_}ro: karo kiro -    -    koro}
\pysrcoutput{k{\_}ri: kari kiri keri kuri kori}
\pysrcoutput{k{\_}pa: kapa -    kepa -    kopa}
\pysrcoutput{k{\_}ra: kara kira kera -    kora}
\pysrcoutput{k{\_}ku: kaku -    -    kuku koku}
\pysrcoutput{k{\_}ki: kaki kiki -    -    koki}}\end{alltt}

Observe in the above example that the context, target, and displayed
material were all based on the lexeme field.  However, the idea of
minimal sets is much more general.  For instance, suppose we wanted to
get a list of wordforms having more than one possible part-of-speech.
Then the target will be part-of-speech field, and the context will be
the lexeme field.  We will also display the English gloss field.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }entries = {[}(e.findtext(\pysrcstring{'lx'}), e.findtext(\pysrcstring{'ps'}), e.findtext(\pysrcstring{'ge'}))
\pysrcmore{... }              \pysrckeyword{for} e \pysrckeyword{in} lexicon
\pysrcmore{... }              \pysrckeyword{if} e.findtext(\pysrcstring{'lx'}) \pysrckeyword{and} e.findtext(\pysrcstring{'ps'}) \pysrckeyword{and} e.findtext(\pysrcstring{'ge'}){]}
\pysrcprompt{>{}>{}> }ms = nltk.MinimalSet((lx, ps{[}0{]}, \pysrcstring{"{\%}s ({\%}s)"} {\%} (ps{[}0{]}, ge))
\pysrcmore{... }              \pysrckeyword{for} (lx, ps, ge) \pysrckeyword{in} entries)
\pysrcprompt{>{}>{}> }\pysrckeyword{for} context \pysrckeyword{in} ms.contexts(){[}:10{]}:
\pysrcmore{... }    \pysrckeyword{print} \pysrcstring{"{\%}10s:"} {\%} context, \pysrcstring{"; "}.join(ms.display{\_}all(context))
\pysrcoutput{  kokovara: N (unripe coconut); V (unripe)}
\pysrcoutput{     kapua: N (sore); V (have sores)}
\pysrcoutput{      koie: N (pig); V (get pig to eat)}
\pysrcoutput{      kovo: C (garden); N (garden); V (work)}
\pysrcoutput{    kavori: N (crayfish); V (collect crayfish or lobster)}
\pysrcoutput{    korita: N (cutlet?); V (dissect meat)}
\pysrcoutput{      keru: N (bone); V (harden like bone)}
\pysrcoutput{  kirokiro: N (bush used for sorcery); V (write)}
\pysrcoutput{    kaapie: N (hook); V (snag)}
\pysrcoutput{       kou: C (heap); V (lay egg)}}\end{alltt}
% Analysing Texts:
% Use the toolbox lexicon to tag a text with part-of-speech information,
% then do POS-sensitive concordance-style searches. 

The following program uses \texttt{MinimalSet} to find pairs of entries in the
corpus which have different attachments based on the \emph{verb} only.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }ms = nltk.MinimalSet()
\pysrcprompt{>{}>{}> }\pysrckeyword{for} entry \pysrckeyword{in} nltk.corpus.ppattach.attachments(\pysrcstring{'training'}):
\pysrcmore{... }    target  = entry.attachment
\pysrcmore{... }    context = (entry.noun1, entry.prep, entry.noun2)
\pysrcmore{... }    display = (target, entry.verb)
\pysrcmore{... }    ms.add(context, target, display)
\pysrcprompt{>{}>{}> }\pysrckeyword{for} context \pysrckeyword{in} ms.contexts():
\pysrcmore{... }    \pysrckeyword{print} context, ms.display{\_}all(context)}\end{alltt}

Here is one of the pairs found by the program.
\begin{itemize}
\item[(185)] \begin{quote}{\ttfamily \raggedright \noindent
\small
received~(NP~offer)~(PP~from~group)~\\
rejected~(NP~offer~(PP~from~group))
}\end{quote}
\end{itemize}

This finding gives us clues to a structural difference: the verb
\docutilsroleexample{receive} usually comes with two following arguments; we receive
something \emph{from} someone.  In contrast, the verb \docutilsroleexample{reject} only
needs a single following argument; we can reject something without
needing to say where it originated from.


%___________________________________________________________________________

\section{Summary}
\label{id138}
\begin{itemize}
\item {} 
diverse motivations for corpus collection

\item {} 
corpus structure, balance, documentation

\item {} 
OLAC

\end{itemize}
% More on utf8? 
% Neil Mayhew: There is another way to force utf-8 output without explicitly encoding
% every string, and that's to 'wrap' the output stream with a codec. This
% technique is particularly useful if you are writing to a file and you
% know that the file *must* have a particular encoding regardless of the
% locale (eg because it's XML).
% f = codecs.open("test.out", "w", "utf-8")
% Then f can be used just like any other file, including print >> f,
% f.write, etc.
% If you really do need to use stdout, you can even wrap that as follows:
% sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
% These things also work for input (with the obvious modifications).
% And, to finish, here's one more nifty trick:
% sys.argv = map(lambda a: a.decode('utf-8'), sys.argv) 


%___________________________________________________________________________

\section{Further Reading}
\label{id139}

\textbf{Shoebox/Toolbox and other tools for field linguistic data management:}
Full details of the Shoebox data format are provided with the distribution \cite{buseman1996shoebox},
and with the latest distribution, freely available from \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}www{\linebreak[0]}.{\linebreak[0]}sil{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}computing{\linebreak[0]}/{\linebreak[0]}toolbox{\linebreak[0]}/}.
Many other software tools support the format.  More examples of our efforts with the format
are documented in \cite{bird1999nels}, \cite{robinson2007toolbox}.
Dozens of other tools for linguistic data management are available, some
surveyed by \cite{bird2003portability}.

\textbf{Some Major Corpora:}
The primary sources of linguistic corpora are the \emph{Linguistic Data Consortium} and
the \emph{European Language Resources Agency}, both with extensive online catalogs.
More details concerning the major corpora mentioned in the chapter are available:
American National Corpus \cite{reppen2005anc},
British National Corpus \cite{bnc1999},
Thesaurus Linguae Graecae \cite{tlg1999},
Child Language Data Exchange System (CHILDES) \cite{macwhinney1995childes},
TIMIT \cite{garofolo1986timit}.
The following papers give accounts of work on corpora that put them to
entirely different uses than were envisaged at the time they were created
\cite{graff2000switchboard}, \cite{cieri2002dasl}.

\textbf{Annotation models and tools:}
An extensive set of models and tools are available, surveyed at
\texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}www{\linebreak[0]}.{\linebreak[0]}exmaralda{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}annotation{\linebreak[0]}/}.
The initial proposal for standoff annotation was \cite{thompson1997standoff}.
The Annotation Graph model was proposed by \cite{bird2001annotation}.

\textbf{Scoring measures:}
Full details of the two scoring methods are available:
Kappa: \cite{carletta1996kappa},
Windowdiff: \cite{pevzner2002windowdiff}.
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
\hypertarget{chap-conclusion}{}\setcounter{chapter}{12}

%___________________________________________________________________________

\chapter{Conclusion}
\label{id155}
\label{chap-conclusion}
% overview of book 


%___________________________________________________________________________

\section{NLP and Intelligence}
\label{nlp-and-intelligence}

A long-standing challenge within computer science has been to build
intelligent machines.  The chief measure of \index{machine intelligence}\textit{machine intelligence} has
been a linguistic one, namely the \index{Turing Test}\textit{Turing Test}: can a dialogue system,
responding to a user's typed input with its own textual output,
perform so naturally that users cannot distinguish it from a human
interlocutor using the same interface?  Today, there is substantial
ongoing research and development in such areas as machine translation
and spoken dialogue, and significant commercial systems are in
widespread use.  The following \index{dialogue}\textit{dialogue} illustrates a typical
application:

\begin{lineblock}{0em}
\item[] S: How may I help you?
\item[] U: When is Saving Private Ryan playing?
\item[] S: For what theater?
\item[] U: The Paramount theater.
\item[] S: Saving Private Ryan is not playing at the Paramount theater, but
\item[] it's playing at the Madison theater at 3:00, 5:30, 8:00, and 10:30.
\end{lineblock}

Today's commercial dialogue systems are strictly limited to
narrowly-defined domains. We could not ask the above system to provide
driving instructions or details of nearby restaurants unless the
requisite information had already been stored and suitable question
and answer sentences had been incorporated into the language
processing system.  Observe that the above system appears to
understand the user's goals: the user asks when a movie is showing and
the system correctly determines from this that the user wants to see
the movie. This inference seems so obvious to humans that we usually
do not even notice it has been made, yet a natural language system
needs to be endowed with this capability in order to interact
naturally. Without it, when asked \docutilsroleexample{Do you know when Saving Private
Ryan is playing}, a system might simply — and unhelpfully
— respond with a cold \docutilsroleexample{Yes}. While it appears that this dialogue
system can perform simple inferences, \index{inference} such sophistication is only
found in cutting edge research prototypes.  Instead, the developers of
commercial dialogue systems use contextual assumptions and simple
business logic to ensure that the different ways in which a user might
express requests or provide information are handled in a way that
makes sense for the particular application.  Thus, whether the user
says \docutilsroleexample{When is ...}, or \docutilsroleexample{I want to know when ...}, or \docutilsroleexample{Can you tell me
when ...}, simple rules will always yield screening times.  This is
sufficient for the system to provide a useful service.

As NLP technologies become more mature, and robust methods for
analysing unrestricted text become more widespread, the prospect of
natural language 'understanding' has re-emerged as a plausible
goal. This has been brought into focus in recent years by a public
'shared task' called Recognizing Textual Entailment (RTE) \cite{dagan2006prt}. The basic
scenario is simple. Let's suppose we are interested in whether we can
find evidence to support a hypothesis such as \docutilsroleexample{Sandra Goudie was
defeated by Max Purnell}. We are given another short text that
appears to be relevant, for example,
\docutilsroleexample{Sandra Goudie was first elected to Parliament in the 2002 elections,
narrowly winning the seat of Coromandel by defeating Labour candidate
Max Purnell and pushing incumbent Green MP Jeanette Fitzsimons into
third place}. The question now is whether the text provides
sufficient evidence for us to accept the hypothesis as true. In this
particular case, the answer is No. This is a conclusion that we can
draw quite easily as humans, but it is very hard to come up with
automated methods for making the right classification. The RTE
Challenges provide data which allow competitors to develop their
systems, but not enough data to allow statistical classifiers to be
trained using standard machine learning techniques. Consequently, some
linguistic analysis is crucial. In the above example, it is important
for the system to note that \docutilsroleexample{Sandra Goudie} names the person being
defeated in the hypothesis, not the person doing the defeating in the
text. As another illustration of the difficulty of the task, consider
the following text/hypothesis pair:
\begin{itemize}
\item {} 
\docutilsroleexample{David Golinkin is the editor or author of eighteen books, and over 150 responsa, articles, sermons and books}

\item {} 
\docutilsroleexample{Golinkin has written eighteen books}

\end{itemize}

In order to determine whether or not the hypothesis is supported by the
text, the system needs at least the following background knowledge:
(i) if someone is an author of a book, then he/she has written that
book; (ii) if someone is an editor of a book, then he/she has not
written that book; (iii) if someone is editor or author of eighteen
books, then he/she is not author of eighteen books.
% discuss another domain, e.g. MT? 

Despite the research-led advances in tasks like RTE, natural language
systems that have been deployed for real-world applications still cannot perform
common-sense reasoning or draw on world knowledge in a general and
robust manner.  We can wait for these difficult artificial
intelligence problems to be solved, but in the meantime it is
necessary to live with some severe limitations on the reasoning and
knowledge capabilities of natural language systems. Accordingly, right
from the beginning, an important goal of NLP research has been to
make progress on the holy grail of natural linguistic interaction
\docutilsroleemphasis{without} recourse to this unrestricted knowledge and reasoning capability.

This has also been the goal of this book, and we hope that readers have
been equipped with the knowledge and skills to build useful NLP systems,
and to advance the field ever closer to its goal of providing
\emph{intelligent} NLP systems.

\part{APPENDICES}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
\hypertarget{appendix-regexps}{}
\setcounter{chapter}{0}
\renewcommand \thechapter{\Alph{chapter}}
\appendix


%___________________________________________________________________________

\chapter{Appendix: Regular Expressions}
\label{appendix-regular-expressions}
\label{appendix-regexps}

This section explores regular expressions in detail, with examples
drawn from language processing.  It builds on the brief overview given
in the introductory programming chapter.

We have already noted that a text can be viewed as a string of
characters. What kinds of processing are performed at the character
level?  Perhaps word games are the most familiar example of such
processing.  In completing a crossword we may want to know which
3-letter English words end with the letter \texttt{c} (e.g.  \texttt{arc}).  We
might want to know how many words can be formed from the letters:
\texttt{a}, \texttt{c}, \texttt{e}, \texttt{o}, and \texttt{n} (e.g. \texttt{ocean}).  We may want
to find out which unique English word contains the substring \texttt{gnt}
(left as an exercise for the reader).  In all these examples, we are
considering which word - drawn from a large set of candidates -
matches a given pattern. To put this in a more computational
framework, we could imagine searching through a large digital corpus
in order to find all words that match a particular pattern. There are
many serious uses of this so-called \emph{pattern matching}.

One instructive example is the task of finding all doubled words in a
text; an example would be the string \texttt{\pysrckeyword{for} \pysrckeyword{for} {\linebreak[0]}example}. Notice that
we would be particularly interested in finding cases where the words
were split across a linebreak (in practice, most erroneously doubled
words occur in this context). Consequently, even with such a
relatively banal task, we need to be able to describe patterns which
refer not just to ''ordinary`` characters, but also to formatting
information.

There are conventions for indicating structure in strings, also known
as \emph{formatting}. For example, there are a number of alternative ways
of formatting a ''date string``, such as \texttt{23{\linebreak[0]}/{\linebreak[0]}06{\linebreak[0]}/{\linebreak[0]}2002}, \texttt{6{\linebreak[0]}/{\linebreak[0]}23{\linebreak[0]}/{\linebreak[0]}02}, or
\texttt{2002{\linebreak[0]}-{\linebreak[0]}06{\linebreak[0]}-{\linebreak[0]}23}.  Whole texts may be formatted, such as an email
message which contains header fields followed by the message
body. Another familiar form of formatting involves visual structure,
such as tabular format and bulleted lists.

Finally, texts may contain explicit ''markup``, such as
\texttt{<{\linebreak[0]}abbrev{\linebreak[0]}>{\linebreak[0]}Phil{\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}abbrev{\linebreak[0]}>}, which provides information about the
interpretation or presentation of some piece of text.  To summarize,
in language processing, strings are ubiquitous, and they often contain
important structure.

So far we have seen elementary examples of pattern matching, the
matching of individual characters.  More often we are interested in
matching \emph{sequences} of characters.  For example, part of the
operation of a naive spell-checker could be to remove a word-final
\texttt{s} from a suspect word token, in case the word is a plural, and see
if the putative singular form exists in the dictionary.  For this we
must locate \texttt{s} and remove it, but only if it precedes a word
boundary.  This requires matching a pattern consisting of two
characters.

Beyond this pattern matching on the \emph{content} of a text, we often want
to process the \emph{formatting} and \emph{markup} of a text.  We may want to
check the formatting of a document (e.g. to ensure that every sentence
begins with a capital letter) or to reformat a document
(e.g. replacing sequences of space characters with a single space).
We may want to find all date strings and extract the year.  We may
want to extract all words contained inside the \texttt{<{\linebreak[0]}abbrev{\linebreak[0]}>{\linebreak[0]} {\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}abbrev{\linebreak[0]}>}
markup in order to construct a list of abbreviations.

Processing the content, format and markup of strings is a central task
in most kinds of NLP.  The most widespread method for string
processing uses \emph{regular expressions}.


%___________________________________________________________________________

\section{Simple Regular Expressions}
\label{simple-regular-expressions}

In this section we will see the building blocks for simple regular
expressions, along with a selection of linguistic examples. We can
think of a regular expression as \emph{a specialized notation for
describing patterns that we want to match}. In order to make explicit
when we are talking about a pattern \emph{patt}, we will
use the notation «\texttt{patt}». The first thing to say
about regular expressions is that most letters match themselves. For
example, the pattern «\texttt{sing}» exactly matches the
string \texttt{sing}. In addition, regular expressions provide us with a
set of \emph{special characters}\footnotemark[2] which give us a way to match \emph{sets of
strings}, and we will now look at these.
\footnotetext[3]{
These are often called \emph{meta-characters}; that is,
characters which express properties of (ordinary) characters.
}


%___________________________________________________________________________

\subsection{The Wildcard}
\label{the-wildcard}

The ''\texttt{.}`` symbol is called a \emph{wildcard}: it matches any single
character. For example, the regular expression «\texttt{s{\linebreak[0]}.{\linebreak[0]}ng}» matches the
following English words: \texttt{sang}, \texttt{sing}, \texttt{song}, and
\texttt{sung}. Note that «\texttt{.}» will match not only alphabetic characters, but
also numeric and whitespace characters. Consequently, «\texttt{s{\linebreak[0]}.{\linebreak[0]}ng}» will also
match non-words such as \texttt{s3ng}.

We can also use the wildcard symbol for counting characters. For
instance «\texttt{.{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}zy}» matches six-letter strings that end in \texttt{zy}.  The
pattern «\texttt{.{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}berry}» finds words like \texttt{cranberry}. In our text from
Wall Street Journal below, the pattern «\texttt{t{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}.}» will match the words
\texttt{that} and \texttt{term}, and will also match the word sequence \texttt{to{\linebreak[0]} {\linebreak[0]}a}
(since the third ''\texttt{.}`` in the pattern can match the space
character):
\begin{quote}{\ttfamily \raggedright \noindent
\small
Paragraph~12~from~wsj{\_}0034:~\\
It's~probably~worth~paying~a~premium~for~funds~that~invest~in~markets~\\
that~are~partially~closed~to~foreign~investors,~such~as~South~Korea,~\\
some~specialists~say.~~But~some~European~funds~recently~have~\\
skyrocketed;~Spain~Fund~has~surged~to~a~startling~120{\%}~premium.~~It~has~\\
been~targeted~by~Japanese~investors~as~a~good~long-term~play~tied~to~\\
1992's~European~economic~integration.~~And~several~new~funds~that~aren't~\\
even~fully~invested~yet~have~jumped~to~trade~at~big~premiums.~\\
~\\
"I'm~very~alarmed~to~see~these~rich~valuations,"~says~Smith~Barney's~\\
Mr.~Porter.
}\end{quote}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Note that the wildcard matches \emph{exactly} one character, and must be
repeated for as many characters as should be matched.  To match a
variable number of characters we must use notation for \emph{optionality}.
\end{boxedminipage}
\end{sffamily}
\end{center}

We can see exactly where a regular expression matches against a string
using NLTK's \texttt{re{\_}show} function.  Readers are encouraged to use
\texttt{re{\_}show} to explore the behavior of regular expressions.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }string = \pysrcstring{"{}"{}"}
\pysrcmore{... }\pysrcstring{It's probably worth paying a premium for funds that invest in markets}
\pysrcmore{... }\pysrcstring{that are partially closed to foreign investors, such as South Korea, ...}
\pysrcmore{... }\pysrcstring{"{}"{}"}
\pysrcprompt{>{}>{}> }nltk.re{\_}show(\pysrcstring{'t...'}, string)
\pysrcoutput{I{\{}t's {\}}probably wor{\{}th p{\}}aying a premium for funds {\{}that{\}} inves{\{}t in{\}} markets}
\pysrcoutput{{\{}that{\}} are par{\{}tial{\}}ly closed {\{}to f{\}}oreign inves{\{}tors{\}}, such as Sou{\{}th K{\}}orea, ...}}\end{alltt}


%___________________________________________________________________________

\subsection{Optionality and Repeatability}
\label{optionality-and-repeatability}

The ''\texttt{?}`` symbol indicates that the immediately preceding regular
expression is optional.  The regular expression «\texttt{colou{\linebreak[0]}?{\linebreak[0]}r}» matches both
British and American spellings, \texttt{colour} and \texttt{color}.  The
expression that precedes the \texttt{?} may be punctuation, such as an
optional hyphen.  For instance «\texttt{e{\linebreak[0]}-{\linebreak[0]}?{\linebreak[0]}mail}» matches both \texttt{e{\linebreak[0]}-{\linebreak[0]}mail} and
\texttt{email}.

The ''\texttt{+}`` symbol indicates that the immediately preceding expression
is repeatable, up to an arbitrary number of times.  For example, the
regular expression «\texttt{coo{\linebreak[0]}+{\linebreak[0]}l}» matches \texttt{cool}, \texttt{coool}, and so on.
This symbol is particularly effective when combined with the \texttt{.}
symbol.  For example, «\texttt{f{\linebreak[0]}.{\linebreak[0]}+{\linebreak[0]}f}» matches all strings of length greater
than two, that begin and end with the letter \texttt{f} (e.g.
\texttt{foolproof}).  The expression «\texttt{.{\linebreak[0]}+{\linebreak[0]}ed}» finds strings that potentially
have the past-tense \texttt{-{\linebreak[0]}ed} suffix.

The ''\texttt{*}`` symbol indicates that the immediately preceding expression
is both optional and repeatable. For example «\texttt{.{\linebreak[0]}*{\linebreak[0]}gnt{\linebreak[0]}.{\linebreak[0]}*}» matches all
strings that contain \texttt{gnt}.

Occasionally we need to match material that spans a line-break.  For
example, we may want to strip out the HTML markup from a document.
To do this we must delete material between angle brackets.
The most obvious expression is: «\texttt{<{\linebreak[0]}.{\linebreak[0]}*{\linebreak[0]}>}».  However, this has two
problems: it will not match an HTML tag that contains a line-break,
and the «\texttt{.{\linebreak[0]}*}» will consume as much material as possible (including the
\texttt{>} character).  To permit matching over a line-break we must use
Python's \texttt{DOTALL} flag, and to ensure that the \texttt{>} matches
against the first instance of the character we must do non-greedy
matching using \texttt{*{\linebreak[0]}?}:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text = \pysrcstring{"{}"{}"one two three <font}
\pysrcmore{... }\pysrcstring{          color="red">four</font> five"{}"{}"}
\pysrcprompt{>{}>{}> }re.sub(r\pysrcstring{'<.*?>'}, \pysrcstring{' '}, text, re.DOTALL)}\end{alltt}


%___________________________________________________________________________

\subsection{Choices}
\label{choices}

Patterns using the wildcard symbol are very effective, but there are
many instances where we want to limit the set of characters that the
wildcard can match.  In such cases we can use the \texttt{{[}{\linebreak[0]}{]}} notation,
which enumerates the set of characters to be matched - this is called
a \emph{character class}.  For example, we can match any English vowel, but
no consonant, using «\texttt{{[}{\linebreak[0]}aeiou{\linebreak[0]}{]}}». Note that this pattern can be
interpreted as saying ''match \texttt{a} or \texttt{e} or ... or \texttt{u}``; that
is, the pattern resembles the wildcard in only matching a string of
length one; unlike the wildcard, it restricts the characters matched
to a specific class (in this case, the vowels).  Note that the order
of vowels in the regular expression is insignificant, and we would
have had the same result with the expression «\texttt{{[}{\linebreak[0]}uoiea{\linebreak[0]}{]}}». As a second
example, the expression «\texttt{p{\linebreak[0]}{[}{\linebreak[0]}aeiou{\linebreak[0]}{]}{\linebreak[0]}t}» matches the words: \texttt{pat},
\texttt{pet}, \texttt{pit}, \texttt{pot}, and \texttt{put}.

We can combine the \texttt{{[}{\linebreak[0]}{]}} notation with our notation for
repeatability.  For example, expression «\texttt{p{\linebreak[0]}{[}{\linebreak[0]}aeiou{\linebreak[0]}{]}{\linebreak[0]}+{\linebreak[0]}t}» matches the words
listed above, along with: \texttt{peat}, \texttt{poet}, and \texttt{pout}.

Often the choices we want to describe cannot be expressed at the level
of individual characters.  As discussed in the tagging tutorial,
different parts of speech are often \emph{tagged} using labels from a
tagset. In the Brown tagset, for example, singular nouns have the tag
\texttt{NN1}, while plural nouns have the tag \texttt{NN2}, while nouns which
are unspecified for number (e.g., \texttt{aircraft}) are tagged \texttt{NN0}. So
we might use «\texttt{NN{\linebreak[0]}.{\linebreak[0]}*}» as a pattern which will match any nominal
tag. Now, suppose we were processing the output of a tagger to extract
string of tokens corresponding to noun phrases, we might want to find
all nouns (\texttt{NN{\linebreak[0]}.{\linebreak[0]}*}), adjectives (\texttt{JJ{\linebreak[0]}.{\linebreak[0]}*}) and determiners (\texttt{DT}),
while excluding all other word types (e.g. verbs \texttt{VB{\linebreak[0]}.{\linebreak[0]}*}).  It is
possible, using a single regular expression, to search for this set of
candidates using the \emph{choice operator} ''\texttt{|}`` as follows:
«\texttt{NN{\linebreak[0]}.{\linebreak[0]}*{\linebreak[0]}|{\linebreak[0]}JJ{\linebreak[0]}.{\linebreak[0]}*{\linebreak[0]}|{\linebreak[0]}DT}». This says: match \texttt{NN{\linebreak[0]}.{\linebreak[0]}*} \emph{or} \texttt{JJ{\linebreak[0]}.{\linebreak[0]}*} \emph{or} \texttt{DT}.

As another example of multi-character choices, suppose that we wanted
to create a program to simplify English prose, replacing rare words
(like \texttt{abode}) with a more frequent, synonymous word (like
\texttt{home}).  In this situation, we need to map from a potentially large
set of words to an individual word.  We can match the set of words
using the choice operator.  In the case of the word \texttt{home}, we would
want to match the regular expression «\texttt{dwelling{\linebreak[0]}|{\linebreak[0]}domicile{\linebreak[0]}|{\linebreak[0]}abode}».
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

Note that the choice operator has wide scope, so that «\texttt{123{\linebreak[0]}|{\linebreak[0]}456}»
is a choice between \texttt{123} and \texttt{456}, and not between \texttt{12356} and
\texttt{12456}.  The latter choice must be written using parentheses: «\texttt{12{\linebreak[0]}({\linebreak[0]}3{\linebreak[0]}|{\linebreak[0]}4{\linebreak[0]}){\linebreak[0]}56}».
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\section{More Complex Regular Expressions}
\label{more-complex-regular-expressions}

In this section we will cover operators which can be used to
construct more powerful and useful regular expressions.


%___________________________________________________________________________

\subsection{Ranges}
\label{ranges}

Earlier we saw how the \texttt{{[}{\linebreak[0]}{]}} notation could be used to express a set
of choices between individual characters.  Instead of listing each
character, it is also possible to express a \emph{range} of characters,
using the \texttt{-} operator.  For example, «\texttt{{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}}» matches any lowercase
letter.  This allows us to avoid the over-permissive matching we noted
above with the pattern «\texttt{t{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}.}». If we were to use the pattern
«\texttt{t{\linebreak[0]}{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}}», then we would no longer match the two word
sequence \texttt{to{\linebreak[0]} {\linebreak[0]}a}.

As expected, ranges can be combined with other operators. For example
«\texttt{{[}{\linebreak[0]}A{\linebreak[0]}-{\linebreak[0]}Z{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}*}» matches words that have an initial capital letter
followed by any number of lowercase letters.  The pattern
«\texttt{20{\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}-{\linebreak[0]}4{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}-{\linebreak[0]}9{\linebreak[0]}{]}}» matches year expressions in the range 2000 to 2049.

Ranges can be combined, e.g.  «\texttt{{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}zA{\linebreak[0]}-{\linebreak[0]}Z{\linebreak[0]}{]}}» which matches any lowercase
or uppercase letter.  The expression «\texttt{{[}{\linebreak[0]}b{\linebreak[0]}-{\linebreak[0]}df{\linebreak[0]}-{\linebreak[0]}hj{\linebreak[0]}-{\linebreak[0]}np{\linebreak[0]}-{\linebreak[0]}tv{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}+}» matches
words consisting only of consonants (e.g. \texttt{pygmy}).


%___________________________________________________________________________

\subsection{Complementation}
\label{complementation}

We just saw that the character class «\texttt{{[}{\linebreak[0]}b{\linebreak[0]}-{\linebreak[0]}df{\linebreak[0]}-{\linebreak[0]}hj{\linebreak[0]}-{\linebreak[0]}np{\linebreak[0]}-{\linebreak[0]}tv{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}+}» allows us to
match sequences of consonants. However, this expression is quite
cumbersome. A better alternative is to say: let's match anything which
isn't a vowel. To do this, we need a way of expressing
\emph{complementation}.  We do this using the symbol ''\texttt{{\textasciicircum}}`` as the first
character inside a class expression \texttt{{[}{\linebreak[0]}{]}}. Let's look at an example.
The regular expression « \texttt{{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]}aeiou{\linebreak[0]}{]}} » is just like our earlier character
class «\texttt{{[}{\linebreak[0]}aeiou{\linebreak[0]}{]}}», except now the set of vowels is preceded
by \texttt{{\textasciicircum}}. The expression as a whole is interpreted as matching
anything which \emph{fails} to match «\texttt{{[}{\linebreak[0]}aeiou{\linebreak[0]}{]}}». In other
words, it matches all lowercase consonants (plus all uppercase letters
and non-alphabetic characters).

As another example, suppose we want to match any string which is
enclosed by the HTML tags for boldface, namely \texttt{<{\linebreak[0]}B{\linebreak[0]}>} and \texttt{<{\linebreak[0]}/{\linebreak[0]}B{\linebreak[0]}>}.
We might try something like this: «\texttt{<{\linebreak[0]}B{\linebreak[0]}>{\linebreak[0]}.{\linebreak[0]}*{\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}B{\linebreak[0]}>}». This would
successfully match \texttt{<{\linebreak[0]}B{\linebreak[0]}>{\linebreak[0]}important{\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}B{\linebreak[0]}>}, but would also match
\texttt{<{\linebreak[0]}B{\linebreak[0]}>{\linebreak[0]}important{\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}B{\linebreak[0]}>{\linebreak[0]} \pysrckeyword{and} {\linebreak[0]}<{\linebreak[0]}B{\linebreak[0]}>{\linebreak[0]}urgent{\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}B{\linebreak[0]}>}, since the «\texttt{.{\linebreak[0]}*}» sub-pattern will
happily match all the characters from the end of \texttt{important} to the
end of \texttt{urgent}. One way of ensuring that we only look at matched
pairs of tags would be to use the expression «\texttt{<{\linebreak[0]}B{\linebreak[0]}>{\linebreak[0]}{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]}<{\linebreak[0]}{]}{\linebreak[0]}*{\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}B{\linebreak[0]}>}», where the character class matches anything other than a left
angle bracket.

Finally, note that character class complementation also works with
ranges. Thus «\texttt{{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}}» matches anything other than the lower case
alphabetic characters \texttt{a} through \texttt{z}.


%___________________________________________________________________________

\subsection{Common Special Symbols}
\label{common-special-symbols}

So far, we have only looked at patterns which match with the content
of character strings. However, it is also useful to be able to refer
to formatting properties of texts. Two important symbols in this
regard are ''\texttt{{\textasciicircum}}`` and ''\texttt{{\$}}`` which are used to \emph{anchor} matches to
the beginnings or ends of lines in a file.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

''\texttt{{\textasciicircum}}`` has two quite distinct uses: it is interpreted as
complementation when it occurs as the first symbol within a character
class, and as matching the beginning of lines when it occurs elsewhere
in a pattern.
\end{boxedminipage}
\end{sffamily}
\end{center}

For example, suppose we wanted to find all the words that occur at the
beginning of lines in the WSJ text above. Our first attempt might look
like «\texttt{{\textasciicircum}{\linebreak[0]}{[}{\linebreak[0]}A{\linebreak[0]}-{\linebreak[0]}Za{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}+}». This says: starting at the beginning of a line,
look for one or more alphabetic characters (upper or lower case),
followed by a space. This will match the words \texttt{that}, \texttt{some},
\texttt{been}, and \texttt{even}. However, it fails to match \texttt{It{\linebreak[0]}'{\linebreak[0]}s}, since
\texttt{'} isn't an alphabetic character. A second attempt might be
«\texttt{{\textasciicircum}{\linebreak[0]}{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]} {\linebreak[0]}{]}{\linebreak[0]}+}», which says to match any string starting at the
beginning of a line, followed by one or more characters which are
\emph{not} the space character, followed by a space. This matches all the
previous words, together with \texttt{It{\linebreak[0]}'{\linebreak[0]}s}, \texttt{skyrocketed}, \texttt{1992s},
\texttt{I{\linebreak[0]}'{\linebreak[0]}m} and \texttt{"{\linebreak[0]}Mr{\linebreak[0]}.}. As a second example, «\texttt{{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}z{\linebreak[0]}{]}{\linebreak[0]}*{\linebreak[0]}s{\linebreak[0]}{\$}}» will
match words ending in \texttt{s} that occur at the end of a line. Finally,
consider the pattern «\texttt{{\textasciicircum}{\linebreak[0]}{\$}}»; this matches strings where no
character occurs between the beginning and the end of a line - in
other words, empty lines!

As we have seen, special characters like ''\texttt{.}``, ''\texttt{*}``, ''\texttt{+}`` and
''\texttt{{\$}}`` give us powerful means to generalize over character
strings. But suppose we wanted to match against a string which itself
contains one or more special characters? An example would be the
arithmetic statement \texttt{{\$}{\linebreak[0]}5{\linebreak[0]}.{\linebreak[0]}00{\linebreak[0]} {\linebreak[0]}*{\linebreak[0]} {\linebreak[0]}({\linebreak[0]}{\$}{\linebreak[0]}3{\linebreak[0]}.{\linebreak[0]}05{\linebreak[0]} {\linebreak[0]}+{\linebreak[0]} {\linebreak[0]}{\$}{\linebreak[0]}0{\linebreak[0]}.{\linebreak[0]}85{\linebreak[0]})}. In this case, we
need to resort to the so-called \emph{escape} character ''\texttt{{\textbackslash}}``
(''backslash``). For example, to match a dollar amount, we might use
«\texttt{{\textbackslash}{\linebreak[0]}{\$}{\linebreak[0]}{[}{\linebreak[0]}1{\linebreak[0]}-{\linebreak[0]}9{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}-{\linebreak[0]}9{\linebreak[0]}{]}{\linebreak[0]}*{\linebreak[0]}{\textbackslash}{\linebreak[0]}.{\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}-{\linebreak[0]}9{\linebreak[0]}{]}{\linebreak[0]}{[}{\linebreak[0]}0{\linebreak[0]}-{\linebreak[0]}9{\linebreak[0]}{]}}».  The same goes for matching
other special characters.
% Advanced Regular Expressions
% zero-width assertions
% more special symbols: \b etc 

\leavevmode
\begin{longtable}[c]{|p{0.145\locallinewidth}|p{0.761\locallinewidth}|}
\hline
\multicolumn{2}{|l|}{
Special Sequences
} \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}b}
 & 
Word boundary (zero width)
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}d}
 & 
Any decimal digit (equivalent to \texttt{{[}{\linebreak[0]}0{\linebreak[0]}-{\linebreak[0]}9{\linebreak[0]}{]}})
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}D}
 & 
Any non-digit character (equivalent to \texttt{{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]}0{\linebreak[0]}-{\linebreak[0]}9{\linebreak[0]}{]}})
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}s}
 & 
Any whitespace character (equivalent to \texttt{{[}{\linebreak[0]} {\linebreak[0]}{\textbackslash}{\linebreak[0]}t{\linebreak[0]}{\textbackslash}{\linebreak[0]}n{\linebreak[0]}{\textbackslash}{\linebreak[0]}r{\linebreak[0]}{\textbackslash}{\linebreak[0]}f{\linebreak[0]}{\textbackslash}{\linebreak[0]}v{\linebreak[0]}{]}}
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}S}
 & 
Any non-whitespace character (equivalent to \texttt{{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]} {\linebreak[0]}{\textbackslash}{\linebreak[0]}t{\linebreak[0]}{\textbackslash}{\linebreak[0]}n{\linebreak[0]}{\textbackslash}{\linebreak[0]}r{\linebreak[0]}{\textbackslash}{\linebreak[0]}f{\linebreak[0]}{\textbackslash}{\linebreak[0]}v{\linebreak[0]}{]}})
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}w}
 & 
Any alphanumeric character (equivalent to \texttt{{[}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}zA{\linebreak[0]}-{\linebreak[0]}Z0{\linebreak[0]}-{\linebreak[0]}9{\_}{\linebreak[0]}{]}})
 \\
\hline

\texttt{{\textbackslash}{\linebreak[0]}W}
 & 
Any non-alphanumeric character (equivalent to \texttt{{[}{\linebreak[0]}{\textasciicircum}{\linebreak[0]}a{\linebreak[0]}-{\linebreak[0]}zA{\linebreak[0]}-{\linebreak[0]}Z0{\linebreak[0]}-{\linebreak[0]}9{\_}{\linebreak[0]}{]}})
 \\
\hline
\caption{}\end{longtable}


%___________________________________________________________________________

\section{Python Interface}
\label{python-interface}

The Python \texttt{re} module provides a convenient interface to an
underlying regular expression engine. The module allows a regular
expression pattern to be compiled into a object whose methods can then
be called.  Let's begin by getting a list of English words:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }wordlist = nltk.corpus.words.words(\pysrcstring{'en'})
\pysrcprompt{>{}>{}> }len(wordlist)
\pysrcoutput{45378}}\end{alltt}

Now we can compile a regular expression for words containing a sequence of two 'a's
and find the matches:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }r1 = re.compile(\pysrcstring{'.*aa.*'})
\pysrcprompt{>{}>{}> }{[}w \pysrckeyword{for} w \pysrckeyword{in} wordlist \pysrckeyword{if} r1.match(w){]}
\pysrcoutput{{[}'Afrikaans', 'bazaar', 'bazaars', 'Canaan', 'Haag', 'Haas', 'Isaac', 'Isaacs', 'Isaacson', 'Izaak', 'Salaam', 'Transvaal', 'Waals'{]}}}\end{alltt}

Suppose now that we want to find all three-letter words ending in the
letter ''\texttt{c}``. Our first attempt might be as follows:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }r1 = re.compile(\pysrcstring{'..c'})
\pysrcprompt{>{}>{}> }{[}w \pysrckeyword{for} w \pysrckeyword{in} wordlist \pysrckeyword{if} r1.match(w){]}{[}:10{]}
\pysrcoutput{{[}'accede', 'acceded', 'accedes', 'accelerate', 'accelerated', 'accelerates', 'accelerating', 'acceleration', 'accelerations', 'accelerator'{]}}}\end{alltt}

The problem is that we have matched words containing three-letter
sequences ending in ''\texttt{c}`` which occur \emph{anywhere within a word}. For
example, the pattern will match ''\texttt{c}`` in words like \texttt{aback},
\texttt{Aerobacter} and \texttt{albacore}.  Instead, we must revise our pattern
so that it is anchored to the beginning and ends of the word: «\texttt{{\textasciicircum}{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}{\$}}»:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }r2 = re.compile(\pysrcstring{'{\textasciicircum}..c{\$}'})
\pysrcprompt{>{}>{}> }{[}w \pysrckeyword{for} w \pysrckeyword{in} wordlist \pysrckeyword{if} r2.match(w){]}
\pysrcoutput{{[}'arc', 'Doc', 'Lac', 'Mac', 'Vic'{]}}}\end{alltt}

In the section on complementation, we briefly looked at the task of
matching strings which were enclosed by HTML markup. Our first attempt
is illustrated in the following code example, where we incorrectly
match the whole string, rather than just the substring
''\texttt{<{\linebreak[0]}B{\linebreak[0]}>{\linebreak[0]}important{\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}B{\linebreak[0]}>}``.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }html = \pysrcstring{'<B>important</B> and <B>urgent</B>'}
\pysrcprompt{>{}>{}> }r2 = re.compile(\pysrcstring{'<B>.*</B>'})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} r2.findall(html)
\pysrcoutput{{[}'<B>important</B> and <B>urgent</B>'{]}}}\end{alltt}

As we pointed out, one solution is to use a character class which
matches with the complement of ''\texttt{<}``:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }r4 = re.compile(\pysrcstring{'<B>{[}{\textasciicircum}<{]}*</B>'})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} r4.findall(html)
\pysrcoutput{{[}'<B>important</B>', '<B>urgent</B>'{]}}}\end{alltt}

However, there is another way of approaching this problem.
«\texttt{<{\linebreak[0]}B{\linebreak[0]}>{\linebreak[0]}.{\linebreak[0]}*{\linebreak[0]}<{\linebreak[0]}/{\linebreak[0]}B{\linebreak[0]}>}» gets the wrong results because the
«\texttt{*}» operator tries to consume as much input as
possible. That is, the matching is said to be \emph{greedy}. In the current
case, «\texttt{*}» matches everything after the first
\texttt{<{\linebreak[0]}B{\linebreak[0]}>}, including the following \texttt{<{\linebreak[0]}/{\linebreak[0]}B{\linebreak[0]}>} and \texttt{<{\linebreak[0]}B{\linebreak[0]}>}.  If we instead
use the non-greedy star operator «\texttt{*{\linebreak[0]}?}», we get the
desired match, since «\texttt{*{\linebreak[0]}?}» tries to consume as little input as possible.


%___________________________________________________________________________

\subsection{Exercises}
\label{id159}
\setcounter{listcnt0}{0}
\begin{list}{\arabic{listcnt0}.}
{
\usecounter{listcnt0}
\addtocounter{listcnt0}{1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
\emph{Pig Latin} is a simple transliteration of English: words starting with
a vowel have \texttt{way} appended (e.g. \texttt{\pysrckeyword{is}} becomes \texttt{isway});
words beginning with a consonant have all consonants up to the first
vowel moved to the end of the word, and then \texttt{ay} is appended
(e.g. \texttt{start} becomes \texttt{artstay}).
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Write a program to convert English text to Pig Latin.

\item {} 
Extend the program to convert text, instead of individual words.

\item {} 
Extend it further to preserve capitalization, to keep \texttt{qu} together
(i.e. so that \texttt{quiet} becomes \texttt{ietquay}), and to detect when \texttt{y}
is used as a consonant (e.g. \texttt{yellow}) vs a vowel (e.g. \texttt{style}).

\end{list}

\item {} 
An interesting challenge for tokenization is words that have been
split across a line-break.  E.g. if \emph{long-term} is split, then we
have the string \texttt{long{\linebreak[0]}-{\linebreak[0]}{\textbackslash}{\linebreak[0]}nterm}.
\setcounter{listcnt1}{0}
\begin{list}{\alph{listcnt1})}
{
\usecounter{listcnt1}
\setlength{\rightmargin}{\leftmargin}
}
\item {} 
Write a regular expression that identifies words that are
hyphenated at a line-break.  The expression will need to include the
\texttt{{\textbackslash}{\linebreak[0]}n} character.

\item {} 
Use \texttt{re{\linebreak[0]}.{\linebreak[0]}sub{\linebreak[0]}({\linebreak[0]})} to remove the \texttt{{\textbackslash}{\linebreak[0]}n} character from these
words.

\end{list}

\item {} 
Write a utility function that takes a URL as its argument, and returns
the contents of the URL, with all HTML markup removed.  Use \texttt{urllib{\linebreak[0]}.{\linebreak[0]}urlopen}
to access the contents of the URL, e.g.
\texttt{raw{\_}contents{\linebreak[0]} {\linebreak[0]}={\linebreak[0]} {\linebreak[0]}urllib{\linebreak[0]}.{\linebreak[0]}urlopen{\linebreak[0]}(\pysrcstring{'{\linebreak[0]}http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}'}){\linebreak[0]}.{\linebreak[0]}read{\linebreak[0]}({\linebreak[0]})}.

\item {} 
Write a program to guess the number of syllables from the orthographic
representation of words (e.g. English text).

\item {} 
Download some text from a language that has vowel harmony (e.g. Hungarian),
extract the vowel sequences of words, and create a vowel bigram table.

\item {} 
Obtain a pronunciation lexicon, and try generating nonsense rhymes.

\end{list}


%___________________________________________________________________________

\section{Further Reading}
\label{id160}

A.M. Kuchling.
\emph{Regular Expression HOWTO},
\href{http://www.amk.ca/python/howto/regex/}{http://www.amk.ca/python/howto/regex/}
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 
% standard global imports
% 
% >>> import nltk, re, pprint 
\hypertarget{appendix-unicode}{}

%___________________________________________________________________________

\chapter{Appendix: Text Processing with Unicode}
\label{appendix-text-processing-with-unicode}
\label{appendix-unicode}

Our programs will often need to deal with different languages, and
different character sets.  The concept of ''plain text`` is a fiction.
If you live in the English-speaking world you probably use ASCII,
possibly without realizing it.  If you live in Europe you might use
one of the extended Latin character sets, containing such characters
as ''ø`` for Danish and Norwegian, ''ő`` for Hungarian,
''ñ`` for Spanish and Breton, and ''ň`` for Czech and
Slovak. In this section, we will give an overview of how to use
Unicode for processing texts that use non-ASCII character sets.


%___________________________________________________________________________

\section{What is Unicode?}
\label{what-is-unicode}

Unicode supports over a million characters.  Each of these
characters is assigned a number, called a \textbf{code point}.  In Python, code
points are written in the form \texttt{{\textbackslash}{\linebreak[0]}u}\emph{XXXX}, where \emph{XXXX} is the number
in 4-digit hexadecimal form.

Within a program, Unicode code points can be manipulated directly, but
when Unicode characters are stored in files or displayed on a terminal
they must be encoded as one or more bytes.  Some encodings (such
as ASCII and Latin-2) use a single byte, so they can only support a
small subset of Unicode, suited to a single language.  Other encodings
(such as UTF-8) use multiple bytes and can represent the full range of
Unicode.

Text in files will be in a particular encoding, so we need some
mechanism for translating it into Unicode — translation into
Unicode is called \textbf{decoding}. Conversely, to write out Unicode to a
file or a terminal, we first need to translate it into a suitable
encoding — this translation out of Unicode is called \textbf{encoding}. The
following diagram illustrates.

\scalebox{0.051429}{\includegraphics{../images/unicode.png}}

From a Unicode perspective, characters are abstract entities which can
be realized as one or more \textbf{glyphs}. Only glyphs can appear on a
screen or be printed on paper. A font is a mapping from characters to glyphs.


%___________________________________________________________________________

\section{Extracting encoded text from files}
\label{extracting-encoded-text-from-files}

Let's assume that we have a small text file, and that we know how it
is encoded. For example, \texttt{polish{\linebreak[0]}-{\linebreak[0]}lat2{\linebreak[0]}.{\linebreak[0]}txt}, as the name suggests, is
a snippet of Polish text (from the Polish Wikipedia; see
\href{http://pl.wikipedia.org/wiki/Biblioteka_Pruska}{http://pl.wikipedia.org/wiki/Biblioteka{\_}Pruska}), encoded as Latin-2,
also known as ISO-8859-2. The function \texttt{nltk{\linebreak[0]}.{\linebreak[0]}data{\linebreak[0]}.{\linebreak[0]}find{\linebreak[0]}({\linebreak[0]})} locates the
file for us.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} nltk.data
\pysrcprompt{>{}>{}> }path = nltk.data.find(\pysrcstring{'samples/polish-lat2.txt'})}\end{alltt}

The Python \texttt{codecs} module provides functions to read encoded data
into Unicode strings, and to write out Unicode strings in encoded
form.  The \texttt{codecs{\linebreak[0]}.{\linebreak[0]}open{\linebreak[0]}({\linebreak[0]})} function takes an encoding parameter to
specify the encoding of the file being read or written. So let's import
the \texttt{codecs} module, and call it with the encoding \texttt{\pysrcstring{'{\linebreak[0]}latin2{\linebreak[0]}'}} to open
our Polish file as Unicode.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} codecs
\pysrcprompt{>{}>{}> }f = codecs.open(path, encoding=\pysrcstring{'latin2'})}\end{alltt}

For a list of encoding parameters allowed by \texttt{codecs}, see
\href{http://docs.python.org/lib/standard-encodings.html}{http://docs.python.org/lib/standard-encodings.html}.

Text read from the file object \texttt{f} will be returned in Unicode. As
we pointed out earlier, in order to view this text on a terminal, we
need to encode it, using a suitable encoding.  The Python-specific
encoding \texttt{unicode{\_}escape} is a dummy encoding that converts all
non-ASCII characters into their \texttt{{\textbackslash}{\linebreak[0]}u}\emph{XXXX} representations.  Code
points above the ASCII 0-127 range but below 256 are represented in
the two-digit form \texttt{{\textbackslash}{\linebreak[0]}x}\emph{XX}.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }lines = f.readlines()
\pysrcprompt{>{}>{}> }\pysrckeyword{for} l \pysrckeyword{in} lines:
\pysrcmore{... }    l = l{[}:-1{]}
\pysrcmore{... }    uni = l.encode(\pysrcstring{'unicode{\_}escape'})
\pysrcmore{... }    \pysrckeyword{print} uni
\pysrcoutput{Pruska Biblioteka Pa{\textbackslash}u0144stwowa. Jej dawne zbiory znane pod nazw{\textbackslash}u0105}
\pysrcoutput{"Berlinka" to skarb kultury i sztuki niemieckiej. Przewiezione przez}
\pysrcoutput{Niemc{\textbackslash}xf3w pod koniec II wojny {\textbackslash}u015bwiatowej na Dolny {\textbackslash}u015al{\textbackslash}u0105sk, zosta{\textbackslash}u0142y}
\pysrcoutput{odnalezione po 1945 r. na terytorium Polski. Trafi{\textbackslash}u0142y do Biblioteki}
\pysrcoutput{Jagiello{\textbackslash}u0144skiej w Krakowie, obejmuj{\textbackslash}u0105 ponad 500 tys. zabytkowych}
\pysrcoutput{archiwali{\textbackslash}xf3w, m.in. manuskrypty Goethego, Mozarta, Beethovena, Bacha.}}\end{alltt}

The first line above illustrates a Unicode escape string, namely
preceded by the \texttt{{\textbackslash}{\linebreak[0]}u} escape string, namely \texttt{{\textbackslash}{\linebreak[0]}u0144} . The relevant
Unicode character will be dislayed on the screen as the glyph
ń.  In the third line of the preceding example, we see
\texttt{{\textbackslash}{\linebreak[0]}xf3}, which corresponds to the glyph ó, and is within the
128-255 range.

In Python, a Unicode string literal can be specified by
preceding an ordinary string literal with a \texttt{u}, as in
\texttt{u\pysrcstring{'{\linebreak[0]}hello{\linebreak[0]}'}}. Arbitrary Unicode characters are defined using the
\texttt{{\textbackslash}{\linebreak[0]}u}\emph{XXXX} escape sequence inside a Unicode string literal.
We find the integer ordinal of a character using \texttt{ord{\linebreak[0]}({\linebreak[0]})}. For example:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }ord(\pysrcstring{'a'})
\pysrcoutput{97}}\end{alltt}

The hexadecimal 4 digit notation for 97 is 0061, so we can define a
Unicode string literal with the appropriate escape sequence:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }a = u\pysrcstring{'{\textbackslash}u0061'}
\pysrcprompt{>{}>{}> }a
\pysrcoutput{u'a'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{print} a
\pysrcoutput{a}}\end{alltt}

Notice that the Python \texttt{\pysrckeyword{print}} statement is assuming a default encoding of the
Unicode character, namely ASCII. However, ń is outside the
ASCII range, so cannot be printed unless we specify an encoding. In
the following example, we have specified that \texttt{\pysrckeyword{print}} should use the
\texttt{repr{\linebreak[0]}({\linebreak[0]})} of the string, which outputs the UTF-8 escape sequences
(of the form \texttt{{\textbackslash}{\linebreak[0]}x}\emph{XX}) rather than trying to render the glyphs.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }nacute = u\pysrcstring{'{\textbackslash}u0144'}
\pysrcprompt{>{}>{}> }nacute
\pysrcoutput{u'{\textbackslash}u0144'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nacute{\_}utf = nacute.encode(\pysrcstring{'utf8'})
\pysrcprompt{>{}>{}> }\pysrckeyword{print} repr(nacute{\_}utf)
\pysrcoutput{'{\textbackslash}xc5{\textbackslash}x84'}}\end{alltt}

If your operating system and locale are set up to render UTF-8 encoded
characters, you ought to be able to give the Python command
\begin{quote}

\texttt{\pysrckeyword{print} {\linebreak[0]}nacute{\_}utf}
\end{quote}

and see ń on your screen.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

There are many factors determining what glyphs are rendered
on your screen. If you are sure that you have the correct encoding,
but your Python code is still failing to produce the glyphs you
expected, you should also check that you have the necessary fonts
installed on your system.
\end{boxedminipage}
\end{sffamily}
\end{center}

The module \texttt{unicodedata} lets us inspect the properties of Unicode
characters. In the following example, we select all characters in the
third line of our Polish text outside the ASCII range and print their
UTF-8 escaped value, followed by their code point integer using the
standard Unicode convention (i.e., prefixing the hex digits with
\texttt{U{\linebreak[0]}+}), followed by their Unicode name.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} unicodedata
\pysrcprompt{>{}>{}> }line = lines{[}2{]}
\pysrcprompt{>{}>{}> }\pysrckeyword{print} line.encode(\pysrcstring{'unicode{\_}escape'})
\pysrcoutput{Niemc{\textbackslash}xf3w pod koniec II wojny {\textbackslash}u015bwiatowej na Dolny {\textbackslash}u015al{\textbackslash}u0105sk, zosta{\textbackslash}u0142y{\textbackslash}n}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{for} c \pysrckeyword{in} line:
\pysrcmore{... }    \pysrckeyword{if} ord(c) > 127:
\pysrcmore{... }        \pysrckeyword{print} \pysrcstring{'{\%}r U+{\%}04x {\%}s'} {\%} (c.encode(\pysrcstring{'utf8'}), ord(c), unicodedata.name(c))
\pysrcoutput{'{\textbackslash}xc3{\textbackslash}xb3' U+00f3 LATIN SMALL LETTER O WITH ACUTE}
\pysrcoutput{'{\textbackslash}xc5{\textbackslash}x9b' U+015b LATIN SMALL LETTER S WITH ACUTE}
\pysrcoutput{'{\textbackslash}xc5{\textbackslash}x9a' U+015a LATIN CAPITAL LETTER S WITH ACUTE}
\pysrcoutput{'{\textbackslash}xc4{\textbackslash}x85' U+0105 LATIN SMALL LETTER A WITH OGONEK}
\pysrcoutput{'{\textbackslash}xc5{\textbackslash}x82' U+0142 LATIN SMALL LETTER L WITH STROKE}}\end{alltt}

If you replace the \texttt{{\%}{\linebreak[0]}r} (which yields the \texttt{repr{\linebreak[0]}({\linebreak[0]})} value) by
\texttt{{\%}{\linebreak[0]}s} in the format string of the code sample above, and if your system supports UTF-8,
you should see an output like the following:

\begin{lineblock}{0em}
\item[] ó U+00f3 LATIN SMALL LETTER O WITH ACUTE
\item[] ś U+015b LATIN SMALL LETTER S WITH ACUTE
\item[] Ś U+015a LATIN CAPITAL LETTER S WITH ACUTE
\item[] ą U+0105 LATIN SMALL LETTER A WITH OGONEK
\item[] ł U+0142 LATIN SMALL LETTER L WITH STROKE
\end{lineblock}

Alternatively, you may need to replace the encoding \texttt{\pysrcstring{'{\linebreak[0]}utf8{\linebreak[0]}'}} in the
example by \texttt{\pysrcstring{'{\linebreak[0]}latin2{\linebreak[0]}'}}, again depending on the details of your system.

The next examples illustrate how Python string methods and the \texttt{re}
module accept Unicode strings.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }line.find(u\pysrcstring{'zosta{\textbackslash}u0142y'})
\pysrcoutput{54}
\pysrcoutput{}\pysrcprompt{>{}>{}> }line = line.lower()
\pysrcprompt{>{}>{}> }\pysrckeyword{print} line.encode(\pysrcstring{'unicode{\_}escape'})
\pysrcoutput{niemc{\textbackslash}xf3w pod koniec ii wojny {\textbackslash}u015bwiatowej na dolny {\textbackslash}u015bl{\textbackslash}u0105sk, zosta{\textbackslash}u0142y{\textbackslash}n}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{import} re
\pysrcprompt{>{}>{}> }m = re.search(u\pysrcstring{'{\textbackslash}u015b{\textbackslash}w*'}, line)
\pysrcprompt{>{}>{}> }m.group()
\pysrcoutput{u'{\textbackslash}u015bwiatowej'}}\end{alltt}

The NLTK \texttt{tokenizer} module allows Unicode strings as input, and
correspondingly yields Unicode strings as output.
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{from} nltk.tokenize \pysrckeyword{import} WordTokenizer
\pysrcprompt{>{}>{}> }tokenizer = WordTokenizer()
\pysrcprompt{>{}>{}> }tokenizer.tokenize(line)  
\pysrcoutput{{[}u'niemc{\textbackslash}xf3w', u'pod', u'koniec', u'ii', u'wojny', u'{\textbackslash}u015bwiatowej',}
\pysrcoutput{u'na', u'dolny', u'{\textbackslash}u015bl{\textbackslash}u0105sk', u'zosta{\textbackslash}u0142y'{]}}}\end{alltt}


%___________________________________________________________________________

\section{Using your local encoding in Python}
\label{using-your-local-encoding-in-python}

If you are used to working with characters in a particular local
encoding, you probably want to be able to use your standard methods
for inputting and editing strings in a Python file. In order to do this,
you need to include the string \texttt{\pysrcstring{'{\linebreak[0]}{\#}{\linebreak[0]} {\linebreak[0]}-{\linebreak[0]}*{\linebreak[0]}-{\linebreak[0]} {\linebreak[0]}coding{\linebreak[0]}:{\linebreak[0]} {\linebreak[0]}<{\linebreak[0]}coding{\linebreak[0]}>{\linebreak[0]} {\linebreak[0]}-{\linebreak[0]}*{\linebreak[0]}-{\linebreak[0]}'}} as the
first or second line of your file. Note that \emph{{\textless}coding{\textgreater}} has to be a
string like \texttt{\pysrcstring{'{\linebreak[0]}latin{\linebreak[0]}-{\linebreak[0]}1{\linebreak[0]}'}}, \texttt{\pysrcstring{'{\linebreak[0]}big5{\linebreak[0]}'}} or \texttt{\pysrcstring{'{\linebreak[0]}utf{\linebreak[0]}-{\linebreak[0]}8{\linebreak[0]}'}}.
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

If you are using Emacs as your editor, the coding specification
will also be interpreted as a specification of the editor's coding
for the file. Not all of the valid Python names for codings are
accepted by Emacs.
\end{boxedminipage}
\end{sffamily}
\end{center}

The following screenshot illustrates the use of UTF-8 encoded string
literals within the IDLE editor:

\scalebox{0.514286}{\includegraphics{../../doc/images/polish-utf8.png}}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

The above example requires that an appropriate font is set in
IDLE's preferences. In this case, we chose Courier CE.
\end{boxedminipage}
\end{sffamily}
\end{center}
% cf http://mail.python.org/pipermail/python-list/2004-February/247783.html
% 
% It is also possible to enter non-ASCII characters in interactive
% mode. IDLE will convert them to the locale's encoding before
% evaluating the source code; if that fails, you get the message you
% see. So where you trying to enter hangul characters in interactive
% mode in a locale that does not support hangul, or are you lacking
% a codec for your locale?
% 
% In interactive mode, UTF-8 is never used (unless you have an
% UTF-8 locale). 

The above example also illustrates how regular expressions can use
encoded strings.


%___________________________________________________________________________

\subsection{Further Reading}
\label{id161}

There are a number of online discussions of Unicode in general, and of
Python facilities for handling Unicode. The following are worth
consulting:
\begin{itemize}
\item {} 
Jason Orendorff, \emph{Unicode for Programmers},
\href{http://www.jorendorff.com/articles/unicode/}{http://www.jorendorff.com/articles/unicode/}.

\item {} 
A. M. Kuchling, \emph{Unicode HOWTO},
\href{http://www.amk.ca/python/howto/unicode}{http://www.amk.ca/python/howto/unicode}

\item {} 
Frederik Lundh, \emph{Python Unicode Objects}, \href{http://effbot.org/zone/unicode-objects.htm}{http://effbot.org/zone/unicode-objects.htm}

\item {} 
Joel Spolsky, \emph{The Absolute Minimum Every Software Developer
Absolutely, Positively Must Know About Unicode and Character Sets
(No Excuses!)}, \href{http://www.joelonsoftware.com/articles/Unicode.html}{http://www.joelonsoftware.com/articles/Unicode.html}

\end{itemize}
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 


%___________________________________________________________________________

\chapter{Appendix: NLP in Python vs other Programming Languages}
\label{appendix-nlp-in-python-vs-other-programming-languages}

Many programming languages have been used for NLP.  As explained in
the Preface, we have chosen Python because we believe it is
well-suited to the special requirements of NLP.  Here we present a
brief survey of several programming languages, for the simple task of
reading a text and printing the words that end with \texttt{ing}.  We begin
with the Python version, which we believe is readily interpretable,
even by non Python programmers:
\begin{quote}{\ttfamily \raggedright \noindent
\small
import~sys~\\
for~line~in~sys.stdin:~\\
~~~~for~word~in~line.split():~\\
~~~~~~~~if~word.endswith('ing'):~\\
~~~~~~~~~~~~print~word
}\end{quote}

Like Python, Perl is a scripting language.  However, its syntax is
obscure.  For instance, it is difficult to guess what kind of entities
are represented by: \texttt{<{\linebreak[0]}>}, \texttt{{\$}}, \texttt{my}, and \texttt{split}, in the following
program:
\begin{quote}{\ttfamily \raggedright \noindent
\small
while~(<>)~{\{}~\\
~~~~foreach~my~{\$}word~(split)~{\{}~\\
~~~~~~~~if~({\$}word~={\textasciitilde}~/ing{\$}/)~{\{}~\\
~~~~~~~~~~~~print~"{\$}word{\textbackslash}n";~\\
~~~~~~~~{\}}~\\
~~~~{\}}~\\
{\}}
}\end{quote}

We agree that ''it is quite easy in Perl to write programs that simply
look like raving gibberish, even to experienced Perl programmers``
(Hammond 2003:47).  Having used Perl ourselves in research and
teaching since the 1980s, we have found that Perl programs of any size
are inordinately difficult to maintain and re-use.  Therefore we
believe Perl is no longer a particularly suitable choice of
programming language for linguists or for language processing.

Prolog is a logic programming language which has been popular for
developing natural language parsers and feature-based grammars, given
the inbuilt support for search and the \emph{unification} operation which
combines two feature structures into one.  Unfortunately Prolog is not
easy to use for string processing or input/output, as the following
program code demonstrates for our linguistic example:
\begin{quote}{\ttfamily \raggedright \noindent
\small
main~:-~\\
~~~~current{\_}input(InputStream),~\\
~~~~read{\_}stream{\_}to{\_}codes(InputStream,~Codes),~\\
~~~~codesToWords(Codes,~Words),~\\
~~~~maplist(string{\_}to{\_}list,~Words,~Strings),~\\
~~~~filter(endsWithIng,~Strings,~MatchingStrings),~\\
~~~~writeMany(MatchingStrings),~\\
~~~~halt.~\\
~\\
codesToWords({[}{]},~{[}{]}).~\\
codesToWords({[}Head~|~Tail{]},~Words)~:-~\\
~~~~(~char{\_}type(Head,~space)~->~\\
~~~~~~~~codesToWords(Tail,~Words)~\\
~~~~;~\\
~~~~~~~~getWord({[}Head~|~Tail{]},~Word,~Rest),~\\
~~~~~~~~codesToWords(Rest,~Words0),~\\
~~~~~~~~Words~=~{[}Word~|~Words0{]}~\\
~~~~).~\\
~\\
getWord({[}{]},~{[}{]},~{[}{]}).~\\
getWord({[}Head~|~Tail{]},~Word,~Rest)~:-~\\
~~~~(~\\
~~~~~~~~(~char{\_}type(Head,~space)~;~char{\_}type(Head,~punct)~)~\\
~~~~->~~Word~=~{[}{]},~Tail~=~Rest~\\
~~~~;~~~getWord(Tail,~Word0,~Rest),~Word~=~{[}Head~|~Word0{]}~\\
~~~~).~\\
~\\
filter(Predicate,~List0,~List)~:-~\\
~~~~(~List0~=~{[}{]}~->~List~=~{[}{]}~\\
~~~~;~~~List0~=~{[}Head~|~Tail{]},~\\
~~~~~~~~(~apply(Predicate,~{[}Head{]})~->~\\
~~~~~~~~~~~~filter(Predicate,~Tail,~List1),~\\
~~~~~~~~~~~~List~=~{[}Head~|~List1{]}~\\
~~~~~~~~;~~~filter(Predicate,~Tail,~List)~\\
~~~~~~~~)~\\
~~~~).~\\
~\\
endsWithIng(String)~:-~sub{\_}string(String,~{\_}Start,~{\_}Len,~0,~'ing').~\\
~\\
writeMany({[}{]}).~\\
writeMany({[}Head~|~Tail{]})~:-~write(Head),~nl,~writeMany(Tail).
}\end{quote}

Java is an object-oriented language incorporating native support for
the Internet, that was originally designed to permit the same
executable program to be run on most computer platforms.  Java has
replaced COBOL as the standard language for business enterprise
software:
\begin{quote}{\ttfamily \raggedright \noindent
\small
import~java.io.*;~\\
public~class~IngWords~{\{}~\\
~~~~public~static~void~main(String{[}{]}~args)~throws~Exception~{\{}~\\
~~~~~~~~BufferedReader~in~=~new~BufferedReader(new~\\
~~~~~~~~~~InputStreamReader(~\\
~~~~~~~~~~~~~~~~~System.in));~\\
~~~~~~~~String~line~=~in.readLine();~\\
~~~~~~~~while~(line~!=~null)~{\{}~\\
~~~~~~~~~~~~for~(String~word~:~line.split("~"))~{\{}~\\
~~~~~~~~~~~~~~~~if~(word.endsWith("ing"))~\\
~~~~~~~~~~~~~~~~~~~~System.out.println(word);~\\
~~~~~~~~~~~~{\}}~\\
~~~~~~~~~~~~line~=~in.readLine();~\\
~~~~~~~~{\}}~\\
~~~~{\}}~\\
{\}}
}\end{quote}

The C programming language is a highly-efficient low-level language
that is popular for operating system and networking software:
\begin{quote}{\ttfamily \raggedright \noindent
\small
{\#}include~<stdio.h>~\\
{\#}include~<string.h>~\\
~\\
int~main(int~argc,~char~**argv)~{\{}~\\
~~~int~i~=~0;~\\
~~~int~c~=~1;~\\
~~~char~buffer{[}1024{]};~\\
~\\
~~~while~(c~!=~EOF)~{\{}~\\
~~~~~~~c~=~fgetc(stdin);~\\
~~~~~~~if~(~(c~>=~'0'~{\&}{\&}~c~<=~'9')~||~(c~>=~'a'~{\&}{\&}~c~<=~'z')~||~(c~>=~'A'~{\&}{\&}~c~<=~'Z')~)~{\{}~\\
~~~~~~~~~~~buffer{[}i++{]}~=~(char)~c;~\\
~~~~~~~~~~~continue;~\\
~~~~~~~{\}}~else~{\{}~\\
~~~~~~~~~~~if~(i~>~2~{\&}{\&}~(strncmp(buffer+i-3,~"ing",~3)~==~0~||~strncmp(buffer+i-3,~"ING",~3)~==~0~)~)~{\{}~\\
~~~~~~~~~~~~~~~buffer{[}i{]}~=~0;~\\
~~~~~~~~~~~~~~~puts(buffer);~\\
~~~~~~~~~~~{\}}~\\
~~~~~~~~~~~i~=~0;~\\
~~~~~~~{\}}~\\
~~~{\}}~\\
~~~return~0;~\\
{\}}
}\end{quote}

LISP is a so-called functional programming language, in which all
objects are lists, and all operations are performed by (nested)
functions of the form \texttt{({\linebreak[0]}function{\linebreak[0]} {\linebreak[0]}arg1{\linebreak[0]} {\linebreak[0]}arg2{\linebreak[0]} {\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]}.{\linebreak[0]})}.  Many of the
earliest NLP systems were implemented in LISP:
\begin{quote}{\ttfamily \raggedright \noindent
\small
(defpackage~"REGEXP-TEST"~(:use~"LISP"~"REGEXP"))~\\
(in-package~"REGEXP-TEST")~\\
~\\
(defun~has-suffix~(string~suffix)~\\
~~"Open~a~file~and~look~for~words~ending~in~{\_}ing."~\\
~~(with-open-file~(f~string)~\\
~~~~~(with-loop-split~(s~f~"~")~\\
~~~~~~~~(mapcar~{\#}'(lambda~(x)~(has{\_}suffix~suffix~x))~s))))~\\
~\\
(defun~has{\_}suffix~(suffix~string)~\\
~~(let*~((suffix{\_}len~(length~suffix))~\\
~~~(string{\_}len~(length~string))~\\
~~~~(base{\_}len~(-~string{\_}len~suffix{\_}len)))~\\
~~~~(if~(string-equal~suffix~string~:start1~0~:end1~NIL~:start2~base{\_}len~:end2~NIL)~\\
~~~~~~~~(print~string))))~\\
~\\
(has-suffix~"test.txt"~"ing")
}\end{quote}

Ruby is a more recently developed scripting language than Python,
best known for its convenient web application framework, \emph{Ruby on Rails}.
Here are two Ruby programs for finding words ending in \docutilsroleexample{ing}
\begin{quote}{\ttfamily \raggedright \noindent
\small
ARGF.each~{\{}~|line|~\\
~~line.split.find{\_}all~{\{}~|word|~\\
~~~~word.match(/ing{\$}/)~\\
~~{\}}.each~{\{}~|word|~\\
~~~~puts~word~\\
~~{\}}~\\
{\}}~\\
~\\
~\\
for~line~in~ARGF~\\
~~for~word~in~line.split~\\
~~~~if~word.match(/ing{\$}/)~then~\\
~~~~~~puts~word~\\
~~~~end~\\
~~end~\\
end
}\end{quote}

Haskell is another functional programming language which permits
a much more compact (but incomprehensible) solution of our simple task:
\begin{quote}{\ttfamily \raggedright \noindent
\small
import~Data.List~\\
main~=~putStr~.~unlines~.~filter~("ing"~`isSuffixOf`)~.~words~=<{}<~getContent
}\end{quote}

The unix shell can also be used for simple linguistic processing.  Here is
a simple pipeline for finding the \docutilsroleexample{ing} words.  The first step transliterates
any whitespace character to a newline, so that each word of the text occurs on
its own line, and the second step finds all lines ending in \docutilsroleexample{ing}
\begin{quote}{\ttfamily \raggedright \noindent
\small
tr~{[}:space:{]}~'{\textbackslash}n'~|~grep~ing{\$}
}\end{quote}

(We are grateful to the following people for furnishing us with these
program samples: Tim Baldwin, Trevor Cohn, David Duke, Rod Farmer,
Andrew Hardie, Aaron Harnly, Edward Ivanovic, and Lars Yencken.)
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 


%___________________________________________________________________________

\chapter{Appendix: NLTK Modules and Corpora}
\label{appendix-nltk-modules-and-corpora}

\textbf{NLTK Organization:}
NLTK is organized into a collection of task-specific packages.  Each
package is a combination of data structures for representing a
particular kind of information such as trees, and implementations of
standard algorithms involving those structures such as parsers.  This
approach is a standard feature of \emph{object-oriented design}, in which
components encapsulate both the resources and methods needed to
accomplish a particular task.

The most fundamental NLTK components are for identifying and
manipulating individual words of text.  These include: \texttt{tokenize},
for breaking up strings of characters into word tokens; \texttt{tag}, for
adding part-of-speech tags, including regular-expression taggers,
n-gram taggers and Brill taggers; and the Porter stemmer.

The second kind of module is for creating and manipulating structured
linguistic information.  These components include: \texttt{tree}, for
representing and processing parse trees; \texttt{featurestructure}, for
building and unifying nested feature structures (or attribute-value
matrices); \texttt{cfg}, for specifying context-free grammars; and \texttt{parse}, for
creating parse trees over input text, including chart parsers, chunk
parsers and probabilistic parsers.

Several utility components are provided to facilitate processing and
visualization.  These include: \texttt{draw}, to visualize NLP structures
and processes; \texttt{probability}, to count and collate events, and
perform statistical estimation; and \texttt{corpora}, to access tagged
linguistic corpora.
% Finally, several advanced components are provided, mostly
% demonstrating NLP applications of machine learning techniques.  These
% include: ``clusterer``, for discovering groups of similar items within
% a large collection, including k-means and expectation maximization;
% ``classifier``, for categorizing text into different types, including
% naive Bayes and maximum entropy; and ``hmm``, for Hidden Markov
% Models, useful for a variety of sequence classification tasks. 

A further group of components is not part of NLTK proper.  These
are a wide selection of third-party contributions, often developed as
student projects at various institutions where NLTK is used, and
distributed in a separate package called \emph{NLTK Contrib}.  Several of
these student contributions, such as the Brill tagger and the HMM
module, have now been incorporated into NLTK.  Although these
contributed components are not maintained, they may serve as a useful starting
point for future student projects.

In addition to software and documentation, NLTK provides substantial
corpus samples.  Many of these can be accessed using the \texttt{corpora}
module, avoiding the need to write specialized file parsing code
before you can do NLP tasks.  These corpora include: Brown Corpus
— 1.15 million words of tagged text in 15 genres; a 10{\%} sample
of the Penn Treebank corpus, consisting of 40,000 words of
syntactically parsed text; a selection of books from Project Gutenberg
totally 1.7 million words; and other corpora for chunking,
prepositional phrase attachment, word-sense disambiguation,
text categorization, and information extraction.

\leavevmode
\begin{longtable}[c]{|p{0.240\locallinewidth}|p{0.189\locallinewidth}|p{0.516\locallinewidth}|}
\hline
\multicolumn{3}{|l|}{\textbf{
Corpora and Corpus Samples Distributed with NLTK
}} \\
\hline
\textbf{
Corpus
} & \textbf{
Compiler
} & \textbf{
Contents
} \\
\hline
\endhead

Alpino Dutch Treebank
 & 
van Noord
 & 
140k words, tagged and parsed (Dutch)
 \\
\hline

Australian ABC News
 & 
Bird
 & 
2 genres, 660k words, sentence-segmented
 \\
\hline

Brown Corpus
 & 
Francis, Kucera
 & 
15 genres, 1.15M words, tagged, categorized
 \\
\hline

CESS-CAT Catalan Treebank
 & 
CLiC-UB et al
 & 
500k words, tagged and parsed
 \\
\hline

CESS-ESP Spanish Treebank
 & 
CLiC-UB et al
 & 
500k words, tagged and parsed
 \\
\hline

CMU Pronouncing Dictionary
 & 
CMU
 & 
127k entries
 \\
\hline

CoNLL 2000 Chunking Data
 & 
Tjong Kim Sang
 & 
270k words, tagged and chunked
 \\
\hline

CoNLL 2002 Named Entity
 & 
Tjong Kim Sang
 & 
700k words, pos- and named-entity-tagged (Dutch, Spanish)
 \\
\hline

Floresta Treebank
 & 
Diana Santos et al
 & 
9k sentences (Portuguese)
 \\
\hline

Genesis Corpus
 & 
Misc web sources
 & 
6 texts, 200k words, 6 languages
 \\
\hline

Gutenberg (sel)
 & 
Hart, Newby, et al
 & 
14 texts, 1.7M words
 \\
\hline

Indian POS-Tagged Corpus
 & 
Kumaran et al
 & 
60k words, tagged (Bangla, Hindi, Marathi, Telugu)
 \\
\hline

MacMorpho Corpus
 & 
NILC, USP, Brazil
 & 
1M words, tagged (Brazilian Portuguese)
 \\
\hline

Movie Reviews
 & 
Pang, Lee
 & 
Sentiment Polarity Dataset 2.0
 \\
\hline

Names Corpus
 & 
Kantrowitz, Ross
 & 
8k male and female names
 \\
\hline

NIST 1999 Info Extr (sel)
 & 
Garofolo
 & 
63k words, newswire and named-entity SGML markup
 \\
\hline

NPS Chat Corpus
 & 
Forsyth, Martell
 & 
10k IM chat posts, POS-tagged and dialogue-act tagged
 \\
\hline

PP Attachment Corpus
 & 
Ratnaparkhi
 & 
28k prepositional phrases, tagged as noun or verb modifiers
 \\
\hline

Presidential Addresses
 & 
Ahrens
 & 
485k words, formatted text
 \\
\hline

Proposition Bank
 & 
Palmer
 & 
113k propositions, 3300 verb frames
 \\
\hline

Question Classification
 & 
Li, Roth
 & 
6k questions, categorized
 \\
\hline

Reuters Corpus
 & 
Reuters
 & 
1.3M words, 10k news documents, categorized
 \\
\hline

Roget's Thesaurus
 & 
Project Gutenberg
 & 
200k words, formatted text
 \\
\hline

RTE Textual Entailment
 & 
Dagan et al
 & 
8k sentence pairs, categorized
 \\
\hline

SEMCOR
 & 
Rus, Mihalcea
 & 
880k words, part-of-speech and sense tagged
 \\
\hline

SENSEVAL 2 Corpus
 & 
Ted Pedersen
 & 
600k words, part-of-speech and sense tagged
 \\
\hline

Shakespeare XML texts (sel)
 & 
Jon Bosak
 & 
8 books
 \\
\hline

Stopwords Corpus
 & 
Porter et al
 & 
2,400 stopwords for 11 languages
 \\
\hline

Switchboard Corpus (sel)
 & 
LDC
 & 
36 phonecalls, transcribed, parsed
 \\
\hline

Univ Decl of Human Rights
 & \begin{itemize}
\item {} 
\end{itemize}
 & 
480k words, 300+ languages
 \\
\hline

US Pres Addr Corpus
 & 
Ahrens
 & 
480k words
 \\
\hline

Penn Treebank (sel)
 & 
LDC
 & 
40k words, tagged and parsed
 \\
\hline

TIMIT Corpus (sel)
 & 
NIST/LDC
 & 
audio files and transcripts for 16 speakers
 \\
\hline

VerbNet 2.1
 & 
Palmer et al
 & 
5k verbs, hierarchically organized, linked to WordNet
 \\
\hline

Wordlist Corpus
 & 
OpenOffice.org et al
 & 
960k words and 20k affixes for 8 languages
 \\
\hline

WordNet 3.0 (English)
 & 
Miller, Fellbaum
 & 
145k synonym sets
 \\
\hline
\caption{}\end{longtable}
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
% -*- mode: rst -*- 
% -*- mode: rst -*- 


%___________________________________________________________________________

\chapter{Appendix: Python and NLTK Cheat Sheet (Draft)}
\label{appendix-python-and-nltk-cheat-sheet-draft}


%___________________________________________________________________________

\section{Python}
\label{id162}


%___________________________________________________________________________

\subsection{Strings}
\label{strings}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }x = \pysrcstring{'Python'}; y = \pysrcstring{'NLTK'}; z = \pysrcstring{'Natural Language Processing'}
\pysrcprompt{>{}>{}> }x + \pysrcstring{'/'} + y
\pysrcoutput{'Python/NLTK'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'LT'} \pysrckeyword{in} y
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }x{[}2:{]}
\pysrcoutput{'thon'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }x{[}::-1{]}
\pysrcoutput{'nohtyP'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }len(x)
\pysrcoutput{6}
\pysrcoutput{}\pysrcprompt{>{}>{}> }z.count(\pysrcstring{'a'})
\pysrcoutput{4}
\pysrcoutput{}\pysrcprompt{>{}>{}> }z.endswith(\pysrcstring{'ing'})
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }z.index(\pysrcstring{'Language'})
\pysrcoutput{8}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'; '}.join({[}x,y,z{]})
\pysrcoutput{'Python; NLTK; Natural Language Processing'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }y.lower()
\pysrcoutput{'nltk'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }z.replace(\pysrcstring{' '}, \pysrcstring{'{\textbackslash}n'})
\pysrcoutput{'Natural{\textbackslash}nLanguage{\textbackslash}nProcessing'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrckeyword{print} z.replace(\pysrcstring{' '}, \pysrcstring{'{\textbackslash}n'})
\pysrcoutput{Natural}
\pysrcoutput{Language}
\pysrcoutput{Processing}
\pysrcoutput{}\pysrcprompt{>{}>{}> }z.split()
\pysrcoutput{{[}'Natural', 'Language', 'Processing'{]}}}\end{alltt}

For more information, type \titlereference{help(str)} at the Python prompt.


%___________________________________________________________________________

\subsection{Lists}
\label{id163}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }x = {[}\pysrcstring{'Natural'}, \pysrcstring{'Language'}{]}; y = {[}\pysrcstring{'Processing'}{]}
\pysrcprompt{>{}>{}> }x{[}0{]}
\pysrcoutput{'Natural'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }list(x{[}0{]})
\pysrcoutput{{[}'N', 'a', 't', 'u', 'r', 'a', 'l'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }x + y
\pysrcoutput{{[}'Natural', 'Language', 'Processing'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'Language'} \pysrckeyword{in} x
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }len(x)
\pysrcoutput{2}
\pysrcoutput{}\pysrcprompt{>{}>{}> }x.index(\pysrcstring{'Language'})
\pysrcoutput{1}}\end{alltt}

The following functions modify the list in-place:
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }x.append(\pysrcstring{'Toolkit'})
\pysrcprompt{>{}>{}> }x
\pysrcoutput{{[}'Natural', 'Language', 'Toolkit'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }x.insert(0, \pysrcstring{'Python'})
\pysrcprompt{>{}>{}> }x
\pysrcoutput{{[}'Python', 'Natural', 'Language', 'Toolkit'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }x.reverse()
\pysrcprompt{>{}>{}> }x
\pysrcoutput{{[}'Toolkit', 'Language', 'Natural', 'Python'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }x.sort()
\pysrcprompt{>{}>{}> }x
\pysrcoutput{{[}'Language', 'Natural', 'Python', 'Toolkit'{]}}}\end{alltt}

For more information, type \titlereference{help(list)} at the Python prompt.


%___________________________________________________________________________

\subsection{Dictionaries}
\label{id164}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }d = {\{}\pysrcstring{'natural'}: \pysrcstring{'adj'}, \pysrcstring{'language'}: \pysrcstring{'noun'}{\}}
\pysrcprompt{>{}>{}> }d{[}\pysrcstring{'natural'}{]}
\pysrcoutput{'adj'}
\pysrcoutput{}\pysrcprompt{>{}>{}> }d{[}\pysrcstring{'toolkit'}{]} = \pysrcstring{'noun'}
\pysrcprompt{>{}>{}> }d
\pysrcoutput{{\{}'natural': 'adj', 'toolkit': 'noun', 'language': 'noun'{\}}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }\pysrcstring{'language'} \pysrckeyword{in} d
\pysrcoutput{True}
\pysrcoutput{}\pysrcprompt{>{}>{}> }d.items()
\pysrcoutput{{[}('natural', 'adj'), ('toolkit', 'noun'), ('language', 'noun'){]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }d.keys()
\pysrcoutput{{[}'natural', 'toolkit', 'language'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }d.values()
\pysrcoutput{{[}'adj', 'noun', 'noun'{]}}}\end{alltt}

For more information, type \titlereference{help(dict)} at the Python prompt.


%___________________________________________________________________________

\subsection{Regular Expressions}
\label{id165}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

to be written
\end{boxedminipage}
\end{sffamily}
\end{center}


%___________________________________________________________________________

\section{NLTK}
\label{nltk}

Many more examples can be found in the NLTK Guides,
available at \texttt{http{\linebreak[0]}:{\linebreak[0]}/{\linebreak[0]}/{\linebreak[0]}nltk{\linebreak[0]}.{\linebreak[0]}org{\linebreak[0]}/{\linebreak[0]}doc{\linebreak[0]}/{\linebreak[0]}guides}.


%___________________________________________________________________________

\subsection{Corpora}
\label{corpora}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }\pysrckeyword{import} nltk
\pysrcprompt{>{}>{}> }dir(nltk.corpus)}\end{alltt}


%___________________________________________________________________________

\subsection{Tokenization}
\label{tokenization}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }text = \pysrcstring{'{}'{}'NLTK, the Natural Language Toolkit, is a suite of program}
\pysrcmore{... }\pysrcstring{modules, data sets and tutorials supporting research and teaching in}
\pysrcmore{... }\pysrcstring{computational linguistics and natural language processing.'{}'{}'}
\pysrcprompt{>{}>{}> }\pysrckeyword{import} nltk
\pysrcprompt{>{}>{}> }nltk.LineTokenizer().tokenize(text)
\pysrcoutput{{[}'NLTK, the Natural Language Toolkit, is a suite of program', 'modules,}
\pysrcoutput{data sets and tutorials supporting research and teaching in', 'computational}
\pysrcoutput{linguistics and natural language processing.'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.WhitespaceTokenizer().tokenize(text)
\pysrcoutput{{[}'NLTK,', 'the', 'Natural', 'Language', 'Toolkit,', 'is', 'a', 'suite',}
\pysrcoutput{ 'of', 'program', 'modules,', 'data', 'sets', 'and', 'tutorials',}
\pysrcoutput{ 'supporting', 'research', 'and', 'teaching', 'in', 'computational',}
\pysrcoutput{ 'linguistics', 'and', 'natural', 'language', 'processing.'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.WordPunctTokenizer().tokenize(text)
\pysrcoutput{{[}'NLTK', ',', 'the', 'Natural', 'Language', 'Toolkit', ',', 'is', 'a',}
\pysrcoutput{ 'suite', 'of', 'program', 'modules', ',', 'data', 'sets', 'and',}
\pysrcoutput{ 'tutorials', 'supporting', 'research', 'and', 'teaching', 'in',}
\pysrcoutput{ 'computational', 'linguistics', 'and', 'natural', 'language',}
\pysrcoutput{ 'processing', '.'{]}}
\pysrcoutput{}\pysrcprompt{>{}>{}> }nltk.RegexpTokenizer(\pysrcstring{', '}, gaps=True).tokenize(text)
\pysrcoutput{{[}'NLTK', 'the Natural Language Toolkit', 'is a suite of program{\textbackslash}nmodules',}
\pysrcoutput{ 'data sets and tutorials supporting research and teaching in{\textbackslash}ncomputational}
\pysrcoutput{ linguistics and natural language processing.'{]}}}\end{alltt}


%___________________________________________________________________________

\subsection{Stemming}
\label{stemming}
\begin{alltt}\small\textbf{\pysrcprompt{>{}>{}> }tokens = nltk.WordPunctTokenizer().tokenize(text)
\pysrcprompt{>{}>{}> }stemmer = nltk.RegexpStemmer(\pysrcstring{'ing{\$}|s{\$}|e{\$}'})
\pysrcprompt{>{}>{}> }\pysrckeyword{for} token \pysrckeyword{in} tokens:
\pysrcmore{... }    \pysrckeyword{print} stemmer.stem(token),
\pysrcoutput{NLTK , th Natural Languag Toolkit , i a suit of program module ,}
\pysrcoutput{data set and tutorial support research and teach in computational}
\pysrcoutput{linguistic and natural languag process .}
\pysrcoutput{}\pysrcprompt{>{}>{}> }stemmer = nltk.PorterStemmer()
\pysrcprompt{>{}>{}> }\pysrckeyword{for} token \pysrckeyword{in} tokens:
\pysrcmore{... }    \pysrckeyword{print} stemmer.stem(token),
\pysrcoutput{NLTK , the Natur Languag Toolkit , is a suit of program modul ,}
\pysrcoutput{data set and tutori support research and teach in comput linguist}
\pysrcoutput{and natur languag process .}}\end{alltt}


%___________________________________________________________________________

\subsection{Tagging}
\label{tagging}
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\textbf{\large Note}
\vspace{2mm}

to be written
\end{boxedminipage}
\end{sffamily}
\end{center}
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
\addcontentsline{toc}{chapter}{Subject Index}
\printindex
\addcontentsline{toc}{chapter}{Subject Index}
\printindex
% -*- mode: rst -*- 
% Footer to be used in all chapters 
\begin{center}\begin{sffamily}
\begin{boxedminipage}{\admonitionwidth}
\vspace{2mm}
\textbf{\large About this document...}
\smallskip

This chapter is a draft from
\emph{Natural Language Processing} {[}\href{http://nltk.org/book.html}{http://nltk.org/book.html}{]},
by \href{http://www.csse.unimelb.edu.au/~sb/}{Steven Bird}, \href{http://www.ltg.ed.ac.uk/~ewan/}{Ewan Klein} and \href{http://www.cis.upenn.edu/~edloper/}{Edward Loper},
Copyright © 2008 the authors.
It is distributed with the \emph{Natural Language Toolkit} {[}\href{http://nltk.org/}{http://nltk.org/}{]},
Version 0.9.5, under the terms of the
\emph{Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States License}
{[}\href{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{http://creativecommons.org/licenses/by-nc-nd/3.0/us/}{]}.

This document is
\end{boxedminipage}
\end{sffamily}
\end{center}
\bibliographystyle{apalike}
\addcontentsline{toc}{chapter}{Bibliography}
\bibliography{../refs}

\end{document}
