60 lines
1.9 KiB
Python
60 lines
1.9 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Natural Language Toolkit: NLTK Command-Line Interface
|
|
#
|
|
# Copyright (C) 2001-2019 NLTK Project
|
|
# URL: <http://nltk.org/>
|
|
# For license information, see LICENSE.TXT
|
|
|
|
|
|
from functools import partial
|
|
from itertools import chain
|
|
from tqdm import tqdm
|
|
|
|
import click
|
|
|
|
from nltk import word_tokenize
|
|
from nltk.util import parallelize_preprocess
|
|
|
|
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
|
|
|
|
|
|
@click.group(context_settings=CONTEXT_SETTINGS)
|
|
@click.version_option()
|
|
def cli():
|
|
pass
|
|
|
|
|
|
@cli.command("tokenize")
|
|
@click.option(
|
|
"--language",
|
|
"-l",
|
|
default="en",
|
|
help="The language for the Punkt sentence tokenization.",
|
|
)
|
|
@click.option(
|
|
"--preserve-line",
|
|
"-l",
|
|
default=True,
|
|
is_flag=True,
|
|
help="An option to keep the preserve the sentence and not sentence tokenize it.",
|
|
)
|
|
@click.option("--processes", "-j", default=1, help="No. of processes.")
|
|
@click.option("--encoding", "-e", default="utf8", help="Specify encoding of file.")
|
|
@click.option(
|
|
"--delimiter", "-d", default=" ", help="Specify delimiter to join the tokens."
|
|
)
|
|
def tokenize_file(language, preserve_line, processes, encoding, delimiter):
|
|
""" This command tokenizes text stream using nltk.word_tokenize """
|
|
with click.get_text_stream("stdin", encoding=encoding) as fin:
|
|
with click.get_text_stream("stdout", encoding=encoding) as fout:
|
|
# If it's single process, joblib parallization is slower,
|
|
# so just process line by line normally.
|
|
if processes == 1:
|
|
for line in tqdm(fin.readlines()):
|
|
print(delimiter.join(word_tokenize(line)), end="\n", file=fout)
|
|
else:
|
|
for outline in parallelize_preprocess(
|
|
word_tokenize, fin.readlines(), processes, progress_bar=True
|
|
):
|
|
print(delimiter.join(outline), end="\n", file=fout)
|