explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-09-21 11:29:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

Match private networks as URLs (#11121)

Browse Source
This commit is contained in:
antonpibm 2022-08-11 12:26:26 +03:00 committed by GitHub
parent 5d54c0e32a
commit 551e73ccfc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spacy/lang/tokenizer_exceptions.py
View File

@ -17,10 +17,6 @@ URL_PATTERN = (
r"(?:\S+(?::\S*)?@)?" r"(?:\S+(?::\S*)?@)?"
r"(?:" r"(?:"
# IP address exclusion # IP address exclusion
# private & local networks
r"(?!(?:10|127)(?:\.\d{1,3}){3})"
r"(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})"
r"(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})"
# IP address dotted notation octets # IP address dotted notation octets
# excludes loopback network 0.0.0.0 # excludes loopback network 0.0.0.0
# excludes reserved space >= 224.0.0.0 # excludes reserved space >= 224.0.0.0

5
spacy/tests/tokenizer/test_urls.py
View File

@ -33,6 +33,9 @@ URLS_SHOULD_MATCH = [
"http://userid:password@example.com/", "http://userid:password@example.com/",
"http://142.42.1.1/", "http://142.42.1.1/",
"http://142.42.1.1:8080/", "http://142.42.1.1:8080/",
"http://10.140.12.13/foo",
"http://10.140.12.13/foo/bar?arg1=baz&arg2=taz",
"http://10.1.1.1",
"http://foo.com/blah_(wikipedia)#cite-1", "http://foo.com/blah_(wikipedia)#cite-1",
"http://foo.com/blah_(wikipedia)_blah#cite-1", "http://foo.com/blah_(wikipedia)_blah#cite-1",
"http://foo.com/unicode_(✪)_in_parens", "http://foo.com/unicode_(✪)_in_parens",
@ -94,6 +97,7 @@ URLS_SHOULD_NOT_MATCH = [
"http://foo.bar/foo(bar)baz quux", "http://foo.bar/foo(bar)baz quux",
"http://-error-.invalid/", "http://-error-.invalid/",
"http://a.b-.co", "http://a.b-.co",
# Loopback and broadcast addresses should be excluded
"http://0.0.0.0", "http://0.0.0.0",
"http://10.1.1.0", "http://10.1.1.0",
"http://10.1.1.255", "http://10.1.1.255",
@ -102,7 +106,6 @@ URLS_SHOULD_NOT_MATCH = [
"http://3628126748", "http://3628126748",
"http://.www.foo.bar/", "http://.www.foo.bar/",
"http://.www.foo.bar./", "http://.www.foo.bar./",
"http://10.1.1.1",
"NASDAQ:GOOG", "NASDAQ:GOOG",
"http://-a.b.co", "http://-a.b.co",
pytest.param("foo.com", marks=pytest.mark.xfail()), pytest.param("foo.com", marks=pytest.mark.xfail()),