rank:0 word: ('i', 96)
rank:1 word: ('python', 86)
rank:2 word: ('ago', 50)
rank:3 word: ('bronze', 36)
rank:4 word: ('have', 29)
rank:5 word: ('×', 25)

>>fromtextblobimportWord>>print(Word("list").definitions)>>['a database containing an ordered array of items (names or topics)',...]

>>fromtextblobimportWord>>print(Word("def").definitions)>>[]

'string'.lower()

fromenumimportEnum,uniquefromtypingimportList,Tuple,Dictimportrequestsfrombs4importBeautifulSoupasbsfromtextblobimportTextBlob,Wordfromtextblob.exceptionsimportNotTranslatedURLS=(('stackoverflow','https://stackoverflow.com/questions/tagged/python'),('AWS','https://aws.amazon.com/lambda/'),('wikipedia AI','https://en.wikipedia.org/wiki/Artificial_intelligence'),('Docker','https://www.docker.com/resources/what-container'),('Python3.8 doc','https://docs.python.org/3/whatsnew/3.8.html'),)PARSER="html.parser"UNACCEPTABLE_LENGTH_OF_WORD=2@uniqueclassPartOfSpeechToLearn(Enum):JJ='Adjective'VB='Verb'NN='Noun'RB='Adverb'defsingle_page_text_from_url(url:str)->str:res=requests.get(url)raw_html=bs(res.text,PARSER)texts_without_html:str=raw_html.textreturntexts_without_htmldeffilter_for_wordbook(morph:TextBlob)->List[str]:word_and_tag:List[Tuple[str,str]]=morph.tagspart_of_speech_to_learn=tuple(pos.nameforposinPartOfSpeechToLearn)filtered_words=[]forwtinword_and_tag:word:str=wt[0].lower()part_of_speech:int=wt[1]ifpart_of_speechnotinpart_of_speech_to_learn:continueifwordinfiltered_words:continueif(len(word)<=UNACCEPTABLE_LENGTH_OF_WORDorlen(Word(word).definitions)==0):continuefiltered_words.append(word)returnfiltered_wordsdefshow_top_50_common_and_uncommon_words(word_and_count:Dict[str,int])->None:words_by_descending_order:List[Tuple[str,int]]=sorted(word_and_count.items(),key=lambdax:x[1],reverse=True)print('the most common words')fori,word_and_countinenumerate(words_by_descending_order[:10]):word:str=word_and_count[0]count:int=word_and_count[1]try:meaning_jp=str(TextBlob(word).translate(to='ja'))exceptNotTranslated:meaning_jp=wordprint(f'rank:{i+1} word: {word} count: {count} meanings: {meaning_jp}')print('the most uncommon words')forj,word_and_countinenumerate(words_by_descending_order[:-11:-1]):word:str=word_and_count[0]count:int=word_and_count[1]try:meaning_jp=str(TextBlob(word).translate(to='ja'))exceptNotTranslated:meaning_jp=wordprint(f'rank:{j+1} word: {word} count: {count} meanings: {meaning_jp}')if__name__=='__main__':word_and_count:Dict[str,int]={}forurlinURLS:print(f'website: {url[0]} url: {url[1]}')texts_without_html=single_page_text_from_url(url[1])morph=TextBlob(texts_without_html)filtered_words=filter_for_wordbook(morph)forwordinfiltered_words:count=morph.words.count(word,case_sensitive=False)ifcount==0:continueifwordnotinword_and_count.keys():word_and_count[word]=countelse:word_and_count[word]+=countprint(f'length of word_and_count: {len(word_and_count)}')show_top_50_common_and_uncommon_words(word_and_count)

deffilter_for_wordbook(morph:TextBlob)->List[str]:word_and_tag:List[Tuple[str,str]]=morph.tagspart_of_speech_to_learn=tuple(pos.nameforposinPartOfSpeechToLearn)filtered_words=[]forwtinword_and_tag:word=wt[0].lower()part_of_speech=wt[1]ifpart_of_speechnotinpart_of_speech_to_learn:continueifwordinfiltered_words:continueif(len(word)<=UNACCEPTABLE_LENGTH_OF_WORDorlen(Word(word).definitions)==0):continuefiltered_words.append(word)returnfiltered_words

if(len(word)<=UNACCEPTABLE_LENGTH_OF_WORDorlen(Word(word).definitions)==0):continue

forwtinword_and_tag:word=wt[0].lower()ifwordinfiltered_words:continue

URLS=(('stackoverflow','https://stackoverflow.com/questions/tagged/python'),('AWS','https://aws.amazon.com/lambda/'),('wikipedia AI','https://en.wikipedia.org/wiki/Artificial_intelligence'),('Docker','https://www.docker.com/resources/what-container'),('Python3.8 doc','https://docs.python.org/3/whatsnew/3.8.html'),)

try:meaning_jp=str(TextBlob(word).translate(to='ja'))exceptNotTranslated:meaning_jp=word

forwordinfiltered_words:count=morph.words.count(word)ifcount==0:continueifwordnotinword_and_count.keys():word_and_count[word]=countelse:word_and_count[word]+=count

count=morph.words.count(word)ifcount==0:continue

the most common words
rank:1 word: intelligence count: 327 meanings: 知性
rank:2 word: artificial count: 280 meanings: 人工的な
rank:3 word: help count: 190 meanings: 助けて
rank:4 word: python count: 186 meanings: python
rank:5 word: error count: 184 meanings: エラー
rank:6 word: target count: 168 meanings: 目標
rank:7 word: new count: 134 meanings: 新着
rank:8 word: learning count: 129 meanings: 学習する
rank:9 word: machine count: 125 meanings: 機械
rank:10 word: have count: 120 meanings: 持ってる

the most uncommon words
rank:1 word: donate count: 1 meanings: 寄付
rank:2 word: math count: 1 meanings: 数学
rank:3 word: inspect count: 1 meanings: 調べる
rank:4 word: ignore count: 1 meanings: 無視する
rank:5 word: processor count: 1 meanings: プロセッサー
rank:6 word: queue count: 1 meanings: キュー
rank:7 word: depend count: 1 meanings: 依存する
rank:8 word: pretend count: 1 meanings: ふり
rank:9 word: decrease count: 1 meanings: 減少
rank:10 word: allocator count: 1 meanings: アロケータ