mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 11:29:13 +03:00
Merge pull request #1288 from geovedi/indonesian
Indonesian language support
This commit is contained in:
Matthew Honnibal
GitHub
commit
2e28982e28
42
spacy/lang/id/__init__.py
Normal file
42
spacy/lang/id/__init__.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .stop_words import STOP_WORDS
|
||||
from .punctuation import TOKENIZER_SUFFIXES, TOKENIZER_PREFIXES, TOKENIZER_INFIXES
|
||||
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
||||
from .norm_exceptions import NORM_EXCEPTIONS
|
||||
from .lemmatizer import LOOKUP
|
||||
from .lex_attrs import LEX_ATTRS
|
||||
from .syntax_iterators import SYNTAX_ITERATORS
|
||||
|
||||
from ..tokenizer_exceptions import BASE_EXCEPTIONS
|
||||
from ...language import Language
|
||||
from ...lemmatizerlookup import Lemmatizer
|
||||
from ...attrs import LANG
|
||||
from ...util import update_exc
|
||||
|
||||
|
||||
class IndonesianDefaults(Language.Defaults):
|
||||
lex_attr_getters = dict(Language.Defaults.lex_attr_getters)
|
||||
lex_attr_getters[LANG] = lambda text: 'id'
|
||||
|
||||
lex_attr_getters.update(LEX_ATTRS)
|
||||
|
||||
tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS)
|
||||
stop_words = set(STOP_WORDS)
|
||||
prefixes = tuple(TOKENIZER_PREFIXES)
|
||||
suffixes = tuple(TOKENIZER_SUFFIXES)
|
||||
infixes = tuple(TOKENIZER_INFIXES)
|
||||
syntax_iterators = dict(SYNTAX_ITERATORS)
|
||||
|
||||
@classmethod
|
||||
def create_lemmatizer(cls, nlp=None):
|
||||
return Lemmatizer(LOOKUP)
|
||||
|
||||
|
||||
class Indonesian(Language):
|
||||
lang = 'id'
|
||||
Defaults = IndonesianDefaults
|
||||
|
||||
|
||||
__all__ = ['Indonesian']
|
||||
3833
spacy/lang/id/_tokenizer_exceptions_list.py
Normal file
3833
spacy/lang/id/_tokenizer_exceptions_list.py
Normal file
File diff suppressed because it is too large
Load Diff
22
spacy/lang/id/examples.py
Normal file
22
spacy/lang/id/examples.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
|
||||
"""
|
||||
Example sentences to test spaCy and its language models.
|
||||
|
||||
>>> from spacy.lang.en.examples import sentences
|
||||
>>> docs = nlp.pipe(sentences)
|
||||
"""
|
||||
|
||||
|
||||
sentences = [
|
||||
"Al Qaidah mengklaim bom mobil yang menewaskan 60 Orang di Mali",
|
||||
"Abu Sayyaf mengeksekusi sandera warga Filipina",
|
||||
"Penyaluran pupuk berasal dari lima lokasi yakni Bontang, Kalimantan Timur, Surabaya, Banyuwangi, Semarang, dan Makassar.",
|
||||
"PT Pupuk Kaltim telah menyalurkan 274.707 ton pupuk bersubsidi ke wilayah penyaluran di 14 provinsi.",
|
||||
"Jakarta adalah kota besar yang nyaris tidak pernah tidur."
|
||||
"Kamu ada di mana semalam?",
|
||||
"Siapa yang membeli makanan ringan tersebut?",
|
||||
"Siapa presiden pertama Republik Indonesia?"
|
||||
]
|
||||
36883
spacy/lang/id/lemmatizer.py
Normal file
36883
spacy/lang/id/lemmatizer.py
Normal file
File diff suppressed because it is too large
Load Diff
42
spacy/lang/id/lex_attrs.py
Normal file
42
spacy/lang/id/lex_attrs.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from ...attrs import LIKE_NUM
|
||||
|
||||
|
||||
_num_words = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven',
|
||||
'eight', 'nine', 'ten', 'eleven', 'twelve', 'thirteen', 'fourteen',
|
||||
'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen', 'twenty',
|
||||
'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety',
|
||||
'hundred', 'thousand', 'million', 'billion', 'trillion', 'quadrillion',
|
||||
'gajillion', 'bazillion',
|
||||
'nol', 'satu', 'dua', 'tiga', 'empat', 'lima', 'enam', 'tujuh',
|
||||
'delapan', 'sembilan', 'sepuluh', 'sebelas', 'duabelas', 'tigabelas',
|
||||
'empatbelas', 'limabelas', 'enambelas', 'tujuhbelas', 'delapanbelas',
|
||||
'sembilanbelas', 'duapuluh', 'seratus', 'seribu', 'sejuta',
|
||||
'ribu', 'rb', 'juta', 'jt', 'miliar', 'biliun', 'triliun',
|
||||
'kuadriliun', 'kuintiliun', 'sekstiliun', 'septiliun', 'oktiliun',
|
||||
'noniliun', 'desiliun',
|
||||
]
|
||||
|
||||
|
||||
def like_num(text):
|
||||
text = text.replace(',', '').replace('.', '')
|
||||
if text.isdigit():
|
||||
return True
|
||||
if text.count('/') == 1:
|
||||
num, denom = text.split('/')
|
||||
if num.isdigit() and denom.isdigit():
|
||||
return True
|
||||
if text in _num_words:
|
||||
return True
|
||||
if text.count('-') == 1:
|
||||
_, num = text.split('-')
|
||||
if num.isdigit() or num in _num_words:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
LEX_ATTRS = {
|
||||
LIKE_NUM: like_num
|
||||
}
|
||||
17
spacy/lang/id/norm_exceptions.py
Normal file
17
spacy/lang/id/norm_exceptions.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
_exc = {
|
||||
"Rp": "$",
|
||||
"IDR": "$",
|
||||
"RMB": "$",
|
||||
"USD": "$",
|
||||
"AUD": "$",
|
||||
"GBP": "$",
|
||||
}
|
||||
|
||||
NORM_EXCEPTIONS = {}
|
||||
|
||||
for string, norm in _exc.items():
|
||||
NORM_EXCEPTIONS[string] = norm
|
||||
NORM_EXCEPTIONS[string.title()] = norm
|
||||
53
spacy/lang/id/punctuation.py
Normal file
53
spacy/lang/id/punctuation.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from ..punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
|
||||
from ..char_classes import merge_chars, split_chars, _currency, _units
|
||||
from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES
|
||||
from ..char_classes import QUOTES, UNITS, ALPHA, ALPHA_LOWER, ALPHA_UPPER, HYPHENS
|
||||
|
||||
_units = (_units + 's bit Gbps Mbps mbps Kbps kbps ƒ ppi px '
|
||||
'Hz kHz MHz GHz mAh '
|
||||
'ratus rb ribu ribuan '
|
||||
'juta jt jutaan mill?iar million bil[l]?iun bilyun billion '
|
||||
)
|
||||
_currency = (_currency + r' USD Rp IDR RMB SGD S\$')
|
||||
_months = ('Januari Februari Maret April Mei Juni Juli Agustus September '
|
||||
'Oktober November Desember January February March May June '
|
||||
'July August October December Jan Feb Mar Jun Jul Aug Sept '
|
||||
'Oct Okt Nov Des ')
|
||||
|
||||
|
||||
UNITS = merge_chars(_units)
|
||||
CURRENCY = merge_chars(_currency)
|
||||
HTML_PREFIX = r'<(b|strong|i|em|p|span|div|br)\s?/>|<a([^>]+)>'
|
||||
HTML_SUFFIX = r'</(b|strong|i|em|p|span|div|a)>'
|
||||
MONTHS = merge_chars(_months)
|
||||
LIST_CURRENCY = split_chars(_currency)
|
||||
|
||||
TOKENIZER_PREFIXES.remove('#') # hashtag
|
||||
_prefixes = TOKENIZER_PREFIXES + LIST_CURRENCY + [HTML_PREFIX] + ['/', '—']
|
||||
|
||||
_suffixes = TOKENIZER_SUFFIXES + [r'\-[Nn]ya', '-[KkMm]u', '[—-]'] + [
|
||||
r'(?<={c})(?:[0-9]+)'.format(c=CURRENCY),
|
||||
r'(?<=[0-9])(?:{u})'.format(u=UNITS),
|
||||
r'(?<=[0-9])%',
|
||||
r'(?<=[0-9{a}]{h})(?:[\.,:-])'.format(a=ALPHA, h=HTML_SUFFIX),
|
||||
r'(?<=[0-9{a}])(?:{h})'.format(a=ALPHA, h=HTML_SUFFIX),
|
||||
]
|
||||
|
||||
_infixes = TOKENIZER_INFIXES + [
|
||||
r'(?<=[0-9])[\\/](?=[0-9%-])',
|
||||
r'(?<=[0-9])%(?=[{a}0-9/])'.format(a=ALPHA),
|
||||
r'(?<={u})[\/-](?=[0-9])'.format(u=UNITS),
|
||||
r'(?<={m})[\/-](?=[0-9])'.format(m=MONTHS),
|
||||
r'(?<=[0-9\)][\.,])"(?=[0-9])',
|
||||
r'(?<=[{a}\)][\.,\'])["—](?=[{a}])'.format(a=ALPHA),
|
||||
r'(?<=[{a}])-(?=[0-9])'.format(a=ALPHA),
|
||||
r'(?<=[0-9])-(?=[{a}])'.format(a=ALPHA),
|
||||
r'(?<=[{a}])[\/-](?={c}{a})'.format(a=ALPHA, c=CURRENCY),
|
||||
]
|
||||
|
||||
TOKENIZER_PREFIXES = _prefixes
|
||||
TOKENIZER_SUFFIXES = _suffixes
|
||||
TOKENIZER_INFIXES = _infixes
|
||||
763
spacy/lang/id/stop_words.py
Normal file
763
spacy/lang/id/stop_words.py
Normal file
|
|
@ -0,0 +1,763 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
STOP_WORDS = set("""
|
||||
ada
|
||||
adalah
|
||||
adanya
|
||||
adapun
|
||||
agak
|
||||
agaknya
|
||||
agar
|
||||
akan
|
||||
akankah
|
||||
akhir
|
||||
akhiri
|
||||
akhirnya
|
||||
aku
|
||||
akulah
|
||||
amat
|
||||
amatlah
|
||||
anda
|
||||
andalah
|
||||
antar
|
||||
antara
|
||||
antaranya
|
||||
apa
|
||||
apaan
|
||||
apabila
|
||||
apakah
|
||||
apalagi
|
||||
apatah
|
||||
artinya
|
||||
asal
|
||||
asalkan
|
||||
atas
|
||||
atau
|
||||
ataukah
|
||||
ataupun
|
||||
awal
|
||||
awalnya
|
||||
bagai
|
||||
bagaikan
|
||||
bagaimana
|
||||
bagaimanakah
|
||||
bagaimanapun
|
||||
bagi
|
||||
bagian
|
||||
bahkan
|
||||
bahwa
|
||||
bahwasanya
|
||||
baik
|
||||
bakal
|
||||
bakalan
|
||||
balik
|
||||
banyak
|
||||
bapak
|
||||
baru
|
||||
bawah
|
||||
beberapa
|
||||
begini
|
||||
beginian
|
||||
beginikah
|
||||
beginilah
|
||||
begitu
|
||||
begitukah
|
||||
begitulah
|
||||
begitupun
|
||||
bekerja
|
||||
belakang
|
||||
belakangan
|
||||
belum
|
||||
belumlah
|
||||
benar
|
||||
benarkah
|
||||
benarlah
|
||||
berada
|
||||
berakhir
|
||||
berakhirlah
|
||||
berakhirnya
|
||||
berapa
|
||||
berapakah
|
||||
berapalah
|
||||
berapapun
|
||||
berarti
|
||||
berawal
|
||||
berbagai
|
||||
berdatangan
|
||||
beri
|
||||
berikan
|
||||
berikut
|
||||
berikutnya
|
||||
berjumlah
|
||||
berkali-kali
|
||||
berkata
|
||||
berkehendak
|
||||
berkeinginan
|
||||
berkenaan
|
||||
berlainan
|
||||
berlalu
|
||||
berlangsung
|
||||
berlebihan
|
||||
bermacam
|
||||
bermacam-macam
|
||||
bermaksud
|
||||
bermula
|
||||
bersama
|
||||
bersama-sama
|
||||
bersiap
|
||||
bersiap-siap
|
||||
bertanya
|
||||
bertanya-tanya
|
||||
berturut
|
||||
berturut-turut
|
||||
bertutur
|
||||
berujar
|
||||
berupa
|
||||
besar
|
||||
betul
|
||||
betulkah
|
||||
biasa
|
||||
biasanya
|
||||
bila
|
||||
bilakah
|
||||
bisa
|
||||
bisakah
|
||||
boleh
|
||||
bolehkah
|
||||
bolehlah
|
||||
buat
|
||||
bukan
|
||||
bukankah
|
||||
bukanlah
|
||||
bukannya
|
||||
bulan
|
||||
bung
|
||||
cara
|
||||
caranya
|
||||
cukup
|
||||
cukupkah
|
||||
cukuplah
|
||||
cuma
|
||||
dahulu
|
||||
dalam
|
||||
dan
|
||||
dapat
|
||||
dari
|
||||
daripada
|
||||
datang
|
||||
dekat
|
||||
demi
|
||||
demikian
|
||||
demikianlah
|
||||
dengan
|
||||
depan
|
||||
di
|
||||
dia
|
||||
diakhiri
|
||||
diakhirinya
|
||||
dialah
|
||||
diantara
|
||||
diantaranya
|
||||
diberi
|
||||
diberikan
|
||||
diberikannya
|
||||
dibuat
|
||||
dibuatnya
|
||||
didapat
|
||||
didatangkan
|
||||
digunakan
|
||||
diibaratkan
|
||||
diibaratkannya
|
||||
diingat
|
||||
diingatkan
|
||||
diinginkan
|
||||
dijawab
|
||||
dijelaskan
|
||||
dijelaskannya
|
||||
dikarenakan
|
||||
dikatakan
|
||||
dikatakannya
|
||||
dikerjakan
|
||||
diketahui
|
||||
diketahuinya
|
||||
dikira
|
||||
dilakukan
|
||||
dilalui
|
||||
dilihat
|
||||
dimaksud
|
||||
dimaksudkan
|
||||
dimaksudkannya
|
||||
dimaksudnya
|
||||
diminta
|
||||
dimintai
|
||||
dimisalkan
|
||||
dimulai
|
||||
dimulailah
|
||||
dimulainya
|
||||
dimungkinkan
|
||||
dini
|
||||
dipastikan
|
||||
diperbuat
|
||||
diperbuatnya
|
||||
dipergunakan
|
||||
diperkirakan
|
||||
diperlihatkan
|
||||
diperlukan
|
||||
diperlukannya
|
||||
dipersoalkan
|
||||
dipertanyakan
|
||||
dipunyai
|
||||
diri
|
||||
dirinya
|
||||
disampaikan
|
||||
disebut
|
||||
disebutkan
|
||||
disebutkannya
|
||||
disini
|
||||
disinilah
|
||||
ditambahkan
|
||||
ditandaskan
|
||||
ditanya
|
||||
ditanyai
|
||||
ditanyakan
|
||||
ditegaskan
|
||||
ditujukan
|
||||
ditunjuk
|
||||
ditunjuki
|
||||
ditunjukkan
|
||||
ditunjukkannya
|
||||
ditunjuknya
|
||||
dituturkan
|
||||
dituturkannya
|
||||
diucapkan
|
||||
diucapkannya
|
||||
diungkapkan
|
||||
dong
|
||||
dua
|
||||
dulu
|
||||
empat
|
||||
enggak
|
||||
enggaknya
|
||||
entah
|
||||
entahlah
|
||||
guna
|
||||
gunakan
|
||||
hal
|
||||
hampir
|
||||
hanya
|
||||
hanyalah
|
||||
hari
|
||||
harus
|
||||
haruslah
|
||||
harusnya
|
||||
hendak
|
||||
hendaklah
|
||||
hendaknya
|
||||
hingga
|
||||
ia
|
||||
ialah
|
||||
ibarat
|
||||
ibaratkan
|
||||
ibaratnya
|
||||
ibu
|
||||
ikut
|
||||
ingat
|
||||
ingat-ingat
|
||||
ingin
|
||||
inginkah
|
||||
inginkan
|
||||
ini
|
||||
inikah
|
||||
inilah
|
||||
itu
|
||||
itukah
|
||||
itulah
|
||||
jadi
|
||||
jadilah
|
||||
jadinya
|
||||
jangan
|
||||
jangankan
|
||||
janganlah
|
||||
jauh
|
||||
jawab
|
||||
jawaban
|
||||
jawabnya
|
||||
jelas
|
||||
jelaskan
|
||||
jelaslah
|
||||
jelasnya
|
||||
jika
|
||||
jikalau
|
||||
juga
|
||||
jumlah
|
||||
jumlahnya
|
||||
justru
|
||||
kala
|
||||
kalau
|
||||
kalaulah
|
||||
kalaupun
|
||||
kalian
|
||||
kami
|
||||
kamilah
|
||||
kamu
|
||||
kamulah
|
||||
kan
|
||||
kapan
|
||||
kapankah
|
||||
kapanpun
|
||||
karena
|
||||
karenanya
|
||||
kasus
|
||||
kata
|
||||
katakan
|
||||
katakanlah
|
||||
katanya
|
||||
ke
|
||||
keadaan
|
||||
kebetulan
|
||||
kecil
|
||||
kedua
|
||||
keduanya
|
||||
keinginan
|
||||
kelamaan
|
||||
kelihatan
|
||||
kelihatannya
|
||||
kelima
|
||||
keluar
|
||||
kembali
|
||||
kemudian
|
||||
kemungkinan
|
||||
kemungkinannya
|
||||
kenapa
|
||||
kepada
|
||||
kepadanya
|
||||
kesampaian
|
||||
keseluruhan
|
||||
keseluruhannya
|
||||
keterlaluan
|
||||
ketika
|
||||
khususnya
|
||||
kini
|
||||
kinilah
|
||||
kira
|
||||
kira-kira
|
||||
kiranya
|
||||
kita
|
||||
kitalah
|
||||
kok
|
||||
kurang
|
||||
lagi
|
||||
lagian
|
||||
lah
|
||||
lain
|
||||
lainnya
|
||||
lalu
|
||||
lama
|
||||
lamanya
|
||||
lanjut
|
||||
lanjutnya
|
||||
lebih
|
||||
lewat
|
||||
lima
|
||||
luar
|
||||
macam
|
||||
maka
|
||||
makanya
|
||||
makin
|
||||
malah
|
||||
malahan
|
||||
mampu
|
||||
mampukah
|
||||
mana
|
||||
manakala
|
||||
manalagi
|
||||
masa
|
||||
masalah
|
||||
masalahnya
|
||||
masih
|
||||
masihkah
|
||||
masing
|
||||
masing-masing
|
||||
mau
|
||||
maupun
|
||||
melainkan
|
||||
melakukan
|
||||
melalui
|
||||
melihat
|
||||
melihatnya
|
||||
memang
|
||||
memastikan
|
||||
memberi
|
||||
memberikan
|
||||
membuat
|
||||
memerlukan
|
||||
memihak
|
||||
meminta
|
||||
memintakan
|
||||
memisalkan
|
||||
memperbuat
|
||||
mempergunakan
|
||||
memperkirakan
|
||||
memperlihatkan
|
||||
mempersiapkan
|
||||
mempersoalkan
|
||||
mempertanyakan
|
||||
mempunyai
|
||||
memulai
|
||||
memungkinkan
|
||||
menaiki
|
||||
menambahkan
|
||||
menandaskan
|
||||
menanti
|
||||
menanti-nanti
|
||||
menantikan
|
||||
menanya
|
||||
menanyai
|
||||
menanyakan
|
||||
mendapat
|
||||
mendapatkan
|
||||
mendatang
|
||||
mendatangi
|
||||
mendatangkan
|
||||
menegaskan
|
||||
mengakhiri
|
||||
mengapa
|
||||
mengatakan
|
||||
mengatakannya
|
||||
mengenai
|
||||
mengerjakan
|
||||
mengetahui
|
||||
menggunakan
|
||||
menghendaki
|
||||
mengibaratkan
|
||||
mengibaratkannya
|
||||
mengingat
|
||||
mengingatkan
|
||||
menginginkan
|
||||
mengira
|
||||
mengucapkan
|
||||
mengucapkannya
|
||||
mengungkapkan
|
||||
menjadi
|
||||
menjawab
|
||||
menjelaskan
|
||||
menuju
|
||||
menunjuk
|
||||
menunjuki
|
||||
menunjukkan
|
||||
menunjuknya
|
||||
menurut
|
||||
menuturkan
|
||||
menyampaikan
|
||||
menyangkut
|
||||
menyatakan
|
||||
menyebutkan
|
||||
menyeluruh
|
||||
menyiapkan
|
||||
merasa
|
||||
mereka
|
||||
merekalah
|
||||
merupakan
|
||||
meski
|
||||
meskipun
|
||||
meyakini
|
||||
meyakinkan
|
||||
minta
|
||||
mirip
|
||||
misal
|
||||
misalkan
|
||||
misalnya
|
||||
mula
|
||||
mulai
|
||||
mulailah
|
||||
mulanya
|
||||
mungkin
|
||||
mungkinkah
|
||||
nah
|
||||
naik
|
||||
namun
|
||||
nanti
|
||||
nantinya
|
||||
nyaris
|
||||
nyatanya
|
||||
oleh
|
||||
olehnya
|
||||
pada
|
||||
padahal
|
||||
padanya
|
||||
pak
|
||||
paling
|
||||
panjang
|
||||
pantas
|
||||
para
|
||||
pasti
|
||||
pastilah
|
||||
penting
|
||||
pentingnya
|
||||
per
|
||||
percuma
|
||||
perlu
|
||||
perlukah
|
||||
perlunya
|
||||
pernah
|
||||
persoalan
|
||||
pertama
|
||||
pertama-tama
|
||||
pertanyaan
|
||||
pertanyakan
|
||||
pihak
|
||||
pihaknya
|
||||
pukul
|
||||
pula
|
||||
pun
|
||||
punya
|
||||
rasa
|
||||
rasanya
|
||||
rata
|
||||
rupanya
|
||||
saat
|
||||
saatnya
|
||||
saja
|
||||
sajalah
|
||||
saling
|
||||
sama
|
||||
sama-sama
|
||||
sambil
|
||||
sampai
|
||||
sampai-sampai
|
||||
sampaikan
|
||||
sana
|
||||
sangat
|
||||
sangatlah
|
||||
satu
|
||||
saya
|
||||
sayalah
|
||||
se
|
||||
sebab
|
||||
sebabnya
|
||||
sebagai
|
||||
sebagaimana
|
||||
sebagainya
|
||||
sebagian
|
||||
sebaik
|
||||
sebaik-baiknya
|
||||
sebaiknya
|
||||
sebaliknya
|
||||
sebanyak
|
||||
sebegini
|
||||
sebegitu
|
||||
sebelum
|
||||
sebelumnya
|
||||
sebenarnya
|
||||
seberapa
|
||||
sebesar
|
||||
sebetulnya
|
||||
sebisanya
|
||||
sebuah
|
||||
sebut
|
||||
sebutlah
|
||||
sebutnya
|
||||
secara
|
||||
secukupnya
|
||||
sedang
|
||||
sedangkan
|
||||
sedemikian
|
||||
sedikit
|
||||
sedikitnya
|
||||
seenaknya
|
||||
segala
|
||||
segalanya
|
||||
segera
|
||||
seharusnya
|
||||
sehingga
|
||||
seingat
|
||||
sejak
|
||||
sejauh
|
||||
sejenak
|
||||
sejumlah
|
||||
sekadar
|
||||
sekadarnya
|
||||
sekali
|
||||
sekali-kali
|
||||
sekalian
|
||||
sekaligus
|
||||
sekalipun
|
||||
sekarang
|
||||
sekarang
|
||||
sekecil
|
||||
seketika
|
||||
sekiranya
|
||||
sekitar
|
||||
sekitarnya
|
||||
sekurang-kurangnya
|
||||
sekurangnya
|
||||
sela
|
||||
selain
|
||||
selaku
|
||||
selalu
|
||||
selama
|
||||
selama-lamanya
|
||||
selamanya
|
||||
selanjutnya
|
||||
seluruh
|
||||
seluruhnya
|
||||
semacam
|
||||
semakin
|
||||
semampu
|
||||
semampunya
|
||||
semasa
|
||||
semasih
|
||||
semata
|
||||
semata-mata
|
||||
semaunya
|
||||
sementara
|
||||
semisal
|
||||
semisalnya
|
||||
sempat
|
||||
semua
|
||||
semuanya
|
||||
semula
|
||||
sendiri
|
||||
sendirian
|
||||
sendirinya
|
||||
seolah
|
||||
seolah-olah
|
||||
seorang
|
||||
sepanjang
|
||||
sepantasnya
|
||||
sepantasnyalah
|
||||
seperlunya
|
||||
seperti
|
||||
sepertinya
|
||||
sepihak
|
||||
sering
|
||||
seringnya
|
||||
serta
|
||||
serupa
|
||||
sesaat
|
||||
sesama
|
||||
sesampai
|
||||
sesegera
|
||||
sesekali
|
||||
seseorang
|
||||
sesuatu
|
||||
sesuatunya
|
||||
sesudah
|
||||
sesudahnya
|
||||
setelah
|
||||
setempat
|
||||
setengah
|
||||
seterusnya
|
||||
setiap
|
||||
setiba
|
||||
setibanya
|
||||
setidak-tidaknya
|
||||
setidaknya
|
||||
setinggi
|
||||
seusai
|
||||
sewaktu
|
||||
siap
|
||||
siapa
|
||||
siapakah
|
||||
siapapun
|
||||
sini
|
||||
sinilah
|
||||
soal
|
||||
soalnya
|
||||
suatu
|
||||
sudah
|
||||
sudahkah
|
||||
sudahlah
|
||||
supaya
|
||||
tadi
|
||||
tadinya
|
||||
tahu
|
||||
tahun
|
||||
tak
|
||||
tambah
|
||||
tambahnya
|
||||
tampak
|
||||
tampaknya
|
||||
tandas
|
||||
tandasnya
|
||||
tanpa
|
||||
tanya
|
||||
tanyakan
|
||||
tanyanya
|
||||
tapi
|
||||
tegas
|
||||
tegasnya
|
||||
telah
|
||||
tempat
|
||||
tengah
|
||||
tentang
|
||||
tentu
|
||||
tentulah
|
||||
tentunya
|
||||
tepat
|
||||
terakhir
|
||||
terasa
|
||||
terbanyak
|
||||
terdahulu
|
||||
terdapat
|
||||
terdiri
|
||||
terhadap
|
||||
terhadapnya
|
||||
teringat
|
||||
teringat-ingat
|
||||
terjadi
|
||||
terjadilah
|
||||
terjadinya
|
||||
terkira
|
||||
terlalu
|
||||
terlebih
|
||||
terlihat
|
||||
termasuk
|
||||
ternyata
|
||||
tersampaikan
|
||||
tersebut
|
||||
tersebutlah
|
||||
tertentu
|
||||
tertuju
|
||||
terus
|
||||
terutama
|
||||
tetap
|
||||
tetapi
|
||||
tiap
|
||||
tiba
|
||||
tiba-tiba
|
||||
tidak
|
||||
tidakkah
|
||||
tidaklah
|
||||
tiga
|
||||
tinggi
|
||||
toh
|
||||
tunjuk
|
||||
turut
|
||||
tutur
|
||||
tuturnya
|
||||
ucap
|
||||
ucapnya
|
||||
ujar
|
||||
ujarnya
|
||||
umum
|
||||
umumnya
|
||||
ungkap
|
||||
ungkapnya
|
||||
untuk
|
||||
usah
|
||||
usai
|
||||
waduh
|
||||
wah
|
||||
wahai
|
||||
waktu
|
||||
waktunya
|
||||
walau
|
||||
walaupun
|
||||
wong
|
||||
yaitu
|
||||
yakin
|
||||
yakni
|
||||
yang
|
||||
""".split())
|
||||
42
spacy/lang/id/syntax_iterators.py
Normal file
42
spacy/lang/id/syntax_iterators.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from ...symbols import NOUN, PROPN, PRON
|
||||
|
||||
|
||||
def noun_chunks(obj):
|
||||
"""
|
||||
Detect base noun phrases from a dependency parse. Works on both Doc and Span.
|
||||
"""
|
||||
labels = ['nsubj', 'nsubj:pass', 'obj', 'iobj', 'ROOT', 'appos', 'nmod', 'nmod:poss']
|
||||
doc = obj.doc # Ensure works on both Doc and Span.
|
||||
np_deps = [doc.vocab.strings[label] for label in labels]
|
||||
conj = doc.vocab.strings.add('conj')
|
||||
np_label = doc.vocab.strings.add('NP')
|
||||
seen = set()
|
||||
for i, word in enumerate(obj):
|
||||
if word.pos not in (NOUN, PROPN, PRON):
|
||||
continue
|
||||
# Prevent nested chunks from being produced
|
||||
if word.i in seen:
|
||||
continue
|
||||
if word.dep in np_deps:
|
||||
if any(w.i in seen for w in word.subtree):
|
||||
continue
|
||||
seen.update(j for j in range(word.left_edge.i, word.right_edge.i+1))
|
||||
yield word.left_edge.i, word.right_edge.i+1, np_label
|
||||
elif word.dep == conj:
|
||||
head = word.head
|
||||
while head.dep == conj and head.head.i < head.i:
|
||||
head = head.head
|
||||
# If the head is an NP, and we're coordinated to it, we're an NP
|
||||
if head.dep in np_deps:
|
||||
if any(w.i in seen for w in word.subtree):
|
||||
continue
|
||||
seen.update(j for j in range(word.left_edge.i, word.right_edge.i+1))
|
||||
yield word.left_edge.i, word.right_edge.i+1, np_label
|
||||
|
||||
|
||||
SYNTAX_ITERATORS = {
|
||||
'noun_chunks': noun_chunks
|
||||
}
|
||||
50
spacy/lang/id/tokenizer_exceptions.py
Normal file
50
spacy/lang/id/tokenizer_exceptions.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
# coding: utf8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import regex as re
|
||||
|
||||
from ._tokenizer_exceptions_list import ID_BASE_EXCEPTIONS
|
||||
from ..tokenizer_exceptions import URL_PATTERN
|
||||
from ...symbols import ORTH
|
||||
|
||||
|
||||
_exc = {}
|
||||
|
||||
for orth in ID_BASE_EXCEPTIONS:
|
||||
_exc[orth] = [{ORTH: orth}]
|
||||
|
||||
orth_title = orth.title()
|
||||
_exc[orth_title] = [{ORTH: orth_title}]
|
||||
|
||||
orth_caps = orth.upper()
|
||||
_exc[orth_caps] = [{ORTH: orth_caps}]
|
||||
|
||||
orth_lower = orth.lower()
|
||||
_exc[orth_lower] = [{ORTH: orth_lower}]
|
||||
|
||||
if '-' in orth:
|
||||
orth_title = '-'.join([part.title() for part in orth.split('-')])
|
||||
_exc[orth_title] = [{ORTH: orth_title}]
|
||||
|
||||
orth_caps = '-'.join([part.upper() for part in orth.split('-')])
|
||||
_exc[orth_caps] = [{ORTH: orth_caps}]
|
||||
|
||||
|
||||
for orth in [
|
||||
"'d", "a.m.", "Adm.", "Bros.", "co.", "Co.", "Corp.", "D.C.", "Dr.", "e.g.",
|
||||
"E.g.", "E.G.", "Gen.", "Gov.", "i.e.", "I.e.", "I.E.", "Inc.", "Jr.",
|
||||
"Ltd.", "Md.", "Messrs.", "Mo.", "Mont.", "Mr.", "Mrs.", "Ms.", "p.m.",
|
||||
"Ph.D.", "Rep.", "Rev.", "Sen.", "St.", "vs.",
|
||||
"B.A.", "B.Ch.E.", "B.Sc.", "Dr.", "Dra.", "Drs.", "Hj.", "Ka.", "Kp.",
|
||||
"M.Ag.", "M.Hum.", "M.Kes,", "M.Kom.", "M.M.", "M.P.", "M.Pd.", "M.Sc.",
|
||||
"M.Si.", "M.Sn.", "M.T.", "M.Th.", "No.", "Pjs.", "Plt.", "R.A.", "S.Ag.",
|
||||
"S.E.", "S.H.", "S.Hut.", "S.K.M.", "S.Kedg.", "S.Kedh.", "S.Kom.",
|
||||
"S.Pd.", "S.Pol.", "S.Psi.", "S.S.", "S.Sos.", "S.T.", "S.Tekp.", "S.Th.",
|
||||
"a.l.", "a.n.", "a.s.", "b.d.", "d.a.", "d.l.", "d/h", "dkk.", "dll.",
|
||||
"dr.", "drh.", "ds.", "dsb.", "dst.", "faks.", "fax.", "hlm.", "i/o",
|
||||
"n.b.", "p.p." "pjs.", "s.d.", "tel.", "u.p.",
|
||||
]:
|
||||
_exc[orth] = [{ORTH: orth}]
|
||||
|
||||
TOKENIZER_EXCEPTIONS = dict(_exc)
|
||||
|
||||
|
|
@ -11,8 +11,8 @@ from ..strings import StringStore
|
|||
from .. import util
|
||||
|
||||
|
||||
_languages = ['bn', 'da', 'de', 'en', 'es', 'fi', 'fr', 'he', 'hu', 'it', 'nb',
|
||||
'nl', 'pl', 'pt', 'sv', 'xx']
|
||||
_languages = ['bn', 'da', 'de', 'en', 'es', 'fi', 'fr', 'he', 'hu', 'id',
|
||||
'it', 'nb', 'nl', 'pl', 'pt', 'sv', 'xx']
|
||||
_models = {'en': ['en_depent_web_sm', 'en_core_web_md'],
|
||||
'de': ['de_core_news_md'],
|
||||
'fr': ['fr_depvec_web_lg'],
|
||||
|
|
@ -86,6 +86,9 @@ def hu_tokenizer():
|
|||
def fi_tokenizer():
|
||||
return util.get_lang_class('fi').Defaults.create_tokenizer()
|
||||
|
||||
@pytest.fixture
|
||||
def id_tokenizer():
|
||||
return util.get_lang_class('id').Defaults.create_tokenizer()
|
||||
|
||||
@pytest.fixture
|
||||
def sv_tokenizer():
|
||||
|
|
|
|||
0
spacy/tests/lang/id/__init__.py
Normal file
0
spacy/tests/lang/id/__init__.py
Normal file
115
spacy/tests/lang/id/test_prefix_suffix_infix.py
Normal file
115
spacy/tests/lang/id/test_prefix_suffix_infix.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
# coding: utf-8
|
||||
"""Test that tokenizer prefixes, suffixes and infixes are handled correctly."""
|
||||
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["(Ma'arif)"])
|
||||
def test_tokenizer_splits_no_special(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["Ma'arif"])
|
||||
def test_tokenizer_splits_no_punct(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["(Ma'arif"])
|
||||
def test_tokenizer_splits_prefix_punct(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["Ma'arif)"])
|
||||
def test_tokenizer_splits_suffix_punct(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["(Ma'arif)"])
|
||||
def test_tokenizer_splits_even_wrap(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["(Ma'arif?)"])
|
||||
def test_tokenizer_splits_uneven_wrap(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text,length', [("S.Kom.", 1), ("SKom.", 2), ("(S.Kom.", 2)])
|
||||
def test_tokenizer_splits_prefix_interact(id_tokenizer, text, length):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == length
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["S.Kom.)"])
|
||||
def test_tokenizer_splits_suffix_interact(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["(S.Kom.)"])
|
||||
def test_tokenizer_splits_even_wrap_interact(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["(S.Kom.?)"])
|
||||
def test_tokenizer_splits_uneven_wrap_interact(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text,length', [("gara-gara", 1), ("Jokowi-Ahok", 3), ("Sukarno-Hatta", 3)])
|
||||
def test_tokenizer_splits_hyphens(id_tokenizer, text, length):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == length
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["0.1-13.5", "0.0-0.1", "103.27-300"])
|
||||
def test_tokenizer_splits_numeric_range(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["ini.Budi", "Halo.Bandung"])
|
||||
def test_tokenizer_splits_period_infix(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["Halo,Bandung", "satu,dua"])
|
||||
def test_tokenizer_splits_comma_infix(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 3
|
||||
assert tokens[0].text == text.split(",")[0]
|
||||
assert tokens[1].text == ","
|
||||
assert tokens[2].text == text.split(",")[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('text', ["halo...Bandung", "dia...pergi"])
|
||||
def test_tokenizer_splits_ellipsis_infix(id_tokenizer, text):
|
||||
tokens = id_tokenizer(text)
|
||||
assert len(tokens) == 3
|
||||
|
||||
|
||||
def test_tokenizer_splits_double_hyphen_infix(id_tokenizer):
|
||||
tokens = id_tokenizer("Arsene Wenger--manajer Arsenal--melakukan konferensi pers.")
|
||||
assert len(tokens) == 10
|
||||
assert tokens[0].text == "Arsene"
|
||||
assert tokens[1].text == "Wenger"
|
||||
assert tokens[2].text == "--"
|
||||
assert tokens[3].text == "manajer"
|
||||
assert tokens[4].text == "Arsenal"
|
||||
assert tokens[5].text == "--"
|
||||
assert tokens[6].text == "melakukan"
|
||||
assert tokens[7].text == "konferensi"
|
||||
assert tokens[8].text == "pers"
|
||||
assert tokens[9].text == "."
|
||||
Loading…
Reference in New Issue
Block a user