1
2
3
4
5
6
7
8
9
10 """
11 Corpus reader for the York-Toronto-Helsinki Parsed Corpus of Old
12 English Prose (YCOE), a 1.5 million word syntactically-annotated
13 corpus of Old English prose texts. The corpus is distributed by the
14 Oxford Text Archive: http://www.ota.ahds.ac.uk/ It is not included
15 with NLTK.
16
17 The YCOE corpus is divided into 100 files, each representing
18 an Old English prose text. Tags used within each text complies
19 to the YCOE standard: http://www-users.york.ac.uk/~lang22/YCOE/YcoeHome.htm
20 """
21
22 from nltk.corpus.reader.util import *
23 from nltk.corpus.reader.api import *
24 from nltk.tokenize import RegexpTokenizer
25 from nltk.corpus.reader.bracket_parse import BracketParseCorpusReader
26 from nltk.corpus.reader.tagged import TaggedCorpusReader
27 from string import split
28 import os, re
29 from nltk.internals import deprecated
30
32 """
33 Corpus reader for the York-Toronto-Helsinki Parsed Corpus of Old
34 English Prose (YCOE), a 1.5 million word syntactically-annotated
35 corpus of Old English prose texts.
36 """
37 - def __init__(self, root, encoding=None):
38 self._psd_reader = YCOEParseCorpusReader(
39 os.path.join(root, 'psd'), '.*', '.psd', encoding=encoding)
40 self._pos_reader = YCOETaggedCorpusReader(
41 os.path.join(root, 'pos'), '.*', '.pos', encoding=encoding)
42
43
44 documents = set(f[:-4] for f in self._psd_reader.files())
45 if set(f[:-4] for f in self._pos_reader.files()) != documents:
46 raise ValueError('Items in "psd" and "pos" '
47 'subdirectories do not match.')
48
49 files = sorted(['%s.psd' % doc for doc in documents] +
50 ['%s.pos' % doc for doc in documents])
51 CorpusReader.__init__(self, root, files, encoding)
52 self._documents = tuple(sorted(documents))
53
55 """
56 Return a list of document identifiers for all documents in
57 this corpus, or for the documents with the given file(s) if
58 specified.
59 """
60 if files is None:
61 return self._documents
62 if isinstance(files, basestring):
63 files = [files]
64 for f in files:
65 if f not in self._files:
66 raise KeyError('File id %s not found' % files)
67
68 return sorted(set(f[:-4] for f in files))
69
70 - def files(self, documents=None):
71 """
72 Return a list of file identifiers for the files that make up
73 this corpus, or that store the given document(s) if specified.
74 """
75 if documents is None:
76 return self._files
77 elif isinstance(documents, basestring):
78 documents = [documents]
79 return sorted(set(['%s.pos' % doc for doc in documents] +
80 ['%s.psd' % doc for doc in documents]))
81
83 """
84 Helper that selects the appropraite files for a given set of
85 documents from a given subcorpus (pos or psd).
86 """
87 if documents is None:
88 documents = self._documents
89 else:
90 if isinstance(documents, basestring):
91 documents = [documents]
92 for document in documents:
93 if document not in self._documents:
94 if document[-4:] in ('.pos', '.psd'):
95 raise ValueError(
96 'Expected a document identifier, not a file '
97 'identifier. (Use corpus.documents() to get '
98 'a list of document identifiers.')
99 else:
100 raise ValueError('Document identifier %s not found'
101 % document)
102 return ['%s.%s' % (d, subcorpus) for d in documents]
103
104
105 - def words(self, documents=None):
107 - def sents(self, documents=None):
109 - def paras(self, documents=None):
119
120
121 @deprecated("Use .raw() or .words() or .tagged_words() or "
122 ".parsed_sents() instead.")
123 - def read(self, items=None, format='parsed'):
130 @deprecated("Use .parsed_sents() instead.")
131 - def parsed(self, items=None):
133 @deprecated("Use .words() instead.")
136 @deprecated("Use .tagged_words() instead.")
137 - def tagged(self, items=None):
139 @deprecated("Operation no longer supported.")
141 raise ValueError('format "chunked" no longer supported')
142
143
145 """Specialized version of the standard bracket parse corpus reader
146 that strips out (CODE ...) and (ID ...) nodes."""
151
153 - def __init__(self, root, items, encoding=None):