import pandas as pd
import numpy as npdf=pd.read_csv('/kaggle/input/islamdata/final data islamophobic (1).csv')
df| Annotation | Unnamed: 1 | Text | |
|---|---|---|---|
| 0 | NaN | 0 | #JusticeForAgraKiBeti\n#Protect_Hindu_Girls \n... |
| 1 | 3.0 | 1 | Group Selfie 🤳 #Apple #AbhinavShukla #Airdrop ... |
| 2 | 3.0 | 2 | News outlets and social media claimed that Meh... |
| 3 | 3.0 | 3 | News outlets and social media claimed that Meh... |
| 4 | 3.0 | 4 | News outlets and social media claimed that Meh... |
| ... | ... | ... | ... |
| 565 | 1.0 | 565 | @IrfanPathan Y fooling around buddy, u want to... |
| 566 | 3.0 | 566 | The Uttar Pradesh Population Bill, 2021 has tr... |
| 567 | 1.0 | 567 | @rockyandmayur @mehartweets @atti_cus @amaanba... |
| 568 | 3.0 | 568 | By treating children with disability as non-ex... |
| 569 | 2.0 | 569 | @rose_k01 @zainabsikander The connection is th... |
570 rows × 3 columns
df['Annotation'].value_counts()3.0 474
1.0 56
2.0 16
33.0 1
Name: Annotation, dtype: int64df.drop('Unnamed: 1',axis=1,inplace=True)df=df.dropna()df.reset_index(drop=True,inplace=True)df.loc[375,'Annotation']=3.0/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:1720: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead
See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
self._setitem_single_column(loc, value, pi)df| Annotation | Text | |
|---|---|---|
| 0 | 3.0 | Group Selfie 🤳 #Apple #AbhinavShukla #Airdrop ... |
| 1 | 3.0 | News outlets and social media claimed that Meh... |
| 2 | 3.0 | News outlets and social media claimed that Meh... |
| 3 | 3.0 | News outlets and social media claimed that Meh... |
| 4 | 3.0 | News outlets and social media claimed that Meh... |
| ... | ... | ... |
| 542 | 1.0 | @IrfanPathan Y fooling around buddy, u want to... |
| 543 | 3.0 | The Uttar Pradesh Population Bill, 2021 has tr... |
| 544 | 1.0 | @rockyandmayur @mehartweets @atti_cus @amaanba... |
| 545 | 3.0 | By treating children with disability as non-ex... |
| 546 | 2.0 | @rose_k01 @zainabsikander The connection is th... |
547 rows × 2 columns
df=df.astype({'Annotation': 'int64'})df.dtypesAnnotation int64
Text object
dtype: objectdf1=pd.read_csv('/kaggle/input/islamdatanew/data hate islamophobic (1).csv')
df1| Unnamed: 0 | Annotation | Text | |
|---|---|---|---|
| 0 | 1.0 | 1 | @CMOKerala more interested in pleaseing #Islam... |
| 1 | 2.0 | 1 | @NijiSachiv But cow dung will sure cure #Quran... |
| 2 | 3.0 | 2 | Perhaps, we could chose...\n#IslamicExtremism ... |
| 3 | 4.0 | 2 | @PetaIndia where is your animal love #Hypocrit... |
| 4 | 5.0 | 1 | Every Islamic terrorist dog must be send to he... |
| ... | ... | ... | ... |
| 322 | 352.0 | 3 | Y'all suck! Yeh I said it.\nOut if the measly ... |
| 323 | 353.0 | 3 | @sureshseshadri1 @prasad_perla @SortedEagle @s... |
| 324 | 354.0 | 1 | @SharjeelUsmani @AfreenFatima136 Anywhere they... |
| 325 | 355.0 | 1 | #Islamicterrorism kill humanity everywhere in ... |
| 326 | NaN | 1 | @SharjeelUsmani @AfreenFatima136 Just coz your... |
327 rows × 3 columns
df1['Annotation'].value_counts()3 214
1 72
2 41
Name: Annotation, dtype: int64df1.drop('Unnamed: 0',axis=1,inplace=True)df1.reset_index(drop=True,inplace=True)df1=df1.dropna()
df1| Annotation | Text | |
|---|---|---|
| 0 | 1 | @CMOKerala more interested in pleaseing #Islam... |
| 1 | 1 | @NijiSachiv But cow dung will sure cure #Quran... |
| 2 | 2 | Perhaps, we could chose...\n#IslamicExtremism ... |
| 3 | 2 | @PetaIndia where is your animal love #Hypocrit... |
| 4 | 1 | Every Islamic terrorist dog must be send to he... |
| ... | ... | ... |
| 322 | 3 | Y'all suck! Yeh I said it.\nOut if the measly ... |
| 323 | 3 | @sureshseshadri1 @prasad_perla @SortedEagle @s... |
| 324 | 1 | @SharjeelUsmani @AfreenFatima136 Anywhere they... |
| 325 | 1 | #Islamicterrorism kill humanity everywhere in ... |
| 326 | 1 | @SharjeelUsmani @AfreenFatima136 Just coz your... |
327 rows × 2 columns
frames=[df,df1]
df_final=pd.concat(frames)
df_final| Annotation | Text | |
|---|---|---|
| 0 | 3 | Group Selfie 🤳 #Apple #AbhinavShukla #Airdrop ... |
| 1 | 3 | News outlets and social media claimed that Meh... |
| 2 | 3 | News outlets and social media claimed that Meh... |
| 3 | 3 | News outlets and social media claimed that Meh... |
| 4 | 3 | News outlets and social media claimed that Meh... |
| ... | ... | ... |
| 322 | 3 | Y'all suck! Yeh I said it.\nOut if the measly ... |
| 323 | 3 | @sureshseshadri1 @prasad_perla @SortedEagle @s... |
| 324 | 1 | @SharjeelUsmani @AfreenFatima136 Anywhere they... |
| 325 | 1 | #Islamicterrorism kill humanity everywhere in ... |
| 326 | 1 | @SharjeelUsmani @AfreenFatima136 Just coz your... |
874 rows × 2 columns
df_final.reset_index(drop=True,inplace=True)df_final['Annotation'].value_counts()3 689
1 128
2 57
Name: Annotation, dtype: int64event_dictionary ={1 : True, 2 : True, 3 : False}
df_final['Islamophobic'] = df_final['Annotation'].map(event_dictionary)df_final['Islamophobic'].value_counts()False 689
True 185
Name: Islamophobic, dtype: int64df_final| Annotation | Text | Islamophobic | |
|---|---|---|---|
| 0 | 3 | Group Selfie 🤳 #Apple #AbhinavShukla #Airdrop ... | False |
| 1 | 3 | News outlets and social media claimed that Meh... | False |
| 2 | 3 | News outlets and social media claimed that Meh... | False |
| 3 | 3 | News outlets and social media claimed that Meh... | False |
| 4 | 3 | News outlets and social media claimed that Meh... | False |
| ... | ... | ... | ... |
| 869 | 3 | Y'all suck! Yeh I said it.\nOut if the measly ... | False |
| 870 | 3 | @sureshseshadri1 @prasad_perla @SortedEagle @s... | False |
| 871 | 1 | @SharjeelUsmani @AfreenFatima136 Anywhere they... | True |
| 872 | 1 | #Islamicterrorism kill humanity everywhere in ... | True |
| 873 | 1 | @SharjeelUsmani @AfreenFatima136 Just coz your... | True |
874 rows × 3 columns
df_final['Tweet']=df_final['Text'].str.lower()df_final.Tweet = df_final.Text.apply(lambda x: re.sub(r'https?:\/\/\S+', '', x))
df_final.Tweet.apply(lambda x: re.sub(r"www\.[a-z]?\.?(com)+|[a-z]+\.(com)", '', x))0 Group Selfie 🤳 #Apple #AbhinavShukla #Airdrop #AEWRevolution2021 #Airdrops #BinanceSmartChain #BSC #ApplePay #Gauahar #JusticeForAgraKiBeti #AuratMarch #AjayDevgn #HappyWomensDay #HalaMadrid #UnacademySachinFilm #Trending #Trendy #KajalAggarwal #InternationalWomensDay
1 News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\n
2 News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\n
3 News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\n
4 News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\n
...
869 Y'all suck! Yeh I said it.\nOut if the measly 82 views, not 1 person had the balls to #retweet & stand against #HamasTerrorists\nALL #TERRORISM for that matter. #Sad\n#IslamicTerrorism will continue to destroy lives. Yours may be next. Its happened before!\n#Remember911 #NeverForget
870 @sureshseshadri1 @prasad_perla @SortedEagle @shitul905 @shytigress @bs_hariharan3 @skbit1990 @Savitritvs @FireFly53647402 @chimnibai @Visalmama @Murali76133156 @nilesh_pat @swetamishraa @keeranl @neerangautam @RashmiShriJS @nair_hena @4mSingh @VivekRanjanSonb @kailashkaushik8 @umeshctyagi @Bhagirath1965 @Abhishe36743271 @Ravisha23624235 @Kaliyugamaya @ResurgentIndia @witnessconscio1 @_ankahi @GeetaRaavi @pramodbansal64 @panditsripathak @raaga31280 @arunp131 @ahindu_ @ArunDeshpande20 @VineshGaba @chandanas @PCheppudira @BesuraTaansane @The_WittyLawyer @abs261 @ignisfatuus1110 @LibtardsKaDoc...
871 @SharjeelUsmani @AfreenFatima136 Anywhere they go..they spread cancer of #terrorism #islamicterrorism
872 #Islamicterrorism kill humanity everywhere in the world #HinduUniteAgainstZehad
873 @SharjeelUsmani @AfreenFatima136 Just coz your religious slogan is war cry of #IslamicTerrorism you are doing information terrorism.. proving other religious slogans to be war cries.. so that you play victim card, escape freely etc. NO, ONLY YOUR RELIGIOUS SLOGAN IS A WAR CRY OF TERRORISTS GLOBALLY!
Name: Tweet, Length: 874, dtype: objectfor i in range(0,len(df_final)):
x=df_final.loc[i,'Text']
y=' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ",x).split())
df_final.loc[i,'Clean']=ydf_final| Annotation | Text | Islamophobic | Tweet | Clean | |
|---|---|---|---|---|---|
| 0 | 3 | Group Selfie 🤳 #Apple #AbhinavShukla #Airdrop #AEWRevolution2021 #Airdrops #BinanceSmartChain #BSC #ApplePay #Gauahar #JusticeForAgraKiBeti #AuratMarch #AjayDevgn #HappyWomensDay #HalaMadrid #UnacademySachinFilm #Trending #Trendy #KajalAggarwal #InternationalWomensDay https://t.co/fGyWerqs8G | False | Group Selfie 🤳 #Apple #AbhinavShukla #Airdrop #AEWRevolution2021 #Airdrops #BinanceSmartChain #BSC #ApplePay #Gauahar #JusticeForAgraKiBeti #AuratMarch #AjayDevgn #HappyWomensDay #HalaMadrid #UnacademySachinFilm #Trending #Trendy #KajalAggarwal #InternationalWomensDay | Group Selfie Apple AbhinavShukla Airdrop AEWRevolution2021 Airdrops BinanceSmartChain BSC ApplePay Gauahar JusticeForAgraKiBeti AuratMarch AjayDevgn HappyWomensDay HalaMadrid UnacademySachinFilm Trending Trendy KajalAggarwal InternationalWomensDay |
| 1 | 3 | News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\nhttps://t.co/1p6VgE2nSI | False | News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\n | News outlets and social media claimed that Mehtab had abducted the girl JusticeForAgraKiBeti was trending on Twitter However the police later clarified that the girl had plotted her own kidnapping with friends AltNewsFactCheck |
| 2 | 3 | News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\nhttps://t.co/MYZ0Eum9LL | False | News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\n | News outlets and social media claimed that Mehtab had abducted the girl JusticeForAgraKiBeti was trending on Twitter However the police later clarified that the girl had plotted her own kidnapping with friends AltNewsFactCheck |
| 3 | 3 | News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\nhttps://t.co/6uKNBmOJVK | False | News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\n | News outlets and social media claimed that Mehtab had abducted the girl JusticeForAgraKiBeti was trending on Twitter However the police later clarified that the girl had plotted her own kidnapping with friends AltNewsFactCheck |
| 4 | 3 | News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\nhttps://t.co/n5f174NelA | False | News outlets and social media claimed that Mehtab had abducted the girl. #JusticeForAgraKiBeti was trending on Twitter. However, the police later clarified that the girl had plotted her own kidnapping with friends. #AltNewsFactCheck | @Priyankajha0 \n\n | News outlets and social media claimed that Mehtab had abducted the girl JusticeForAgraKiBeti was trending on Twitter However the police later clarified that the girl had plotted her own kidnapping with friends AltNewsFactCheck |
| ... | ... | ... | ... | ... | ... |
| 869 | 3 | Y'all suck! Yeh I said it.\nOut if the measly 82 views, not 1 person had the balls to #retweet & stand against #HamasTerrorists\nALL #TERRORISM for that matter. #Sad\n#IslamicTerrorism will continue to destroy lives. Yours may be next. Its happened before!\n#Remember911 #NeverForget https://t.co/a2GRWAnrKz | False | Y'all suck! Yeh I said it.\nOut if the measly 82 views, not 1 person had the balls to #retweet & stand against #HamasTerrorists\nALL #TERRORISM for that matter. #Sad\n#IslamicTerrorism will continue to destroy lives. Yours may be next. Its happened before!\n#Remember911 #NeverForget | Y all suck Yeh I said it Out if the measly 82 views not 1 person had the balls to retweet amp stand against HamasTerrorists ALL TERRORISM for that matter Sad IslamicTerrorism will continue to destroy lives Yours may be next Its happened before Remember911 NeverForget |
| 870 | 3 | @sureshseshadri1 @prasad_perla @SortedEagle @shitul905 @shytigress @bs_hariharan3 @skbit1990 @Savitritvs @FireFly53647402 @chimnibai @Visalmama @Murali76133156 @nilesh_pat @swetamishraa @keeranl @neerangautam @RashmiShriJS @nair_hena @4mSingh @VivekRanjanSonb @kailashkaushik8 @umeshctyagi @Bhagirath1965 @Abhishe36743271 @Ravisha23624235 @Kaliyugamaya @ResurgentIndia @witnessconscio1 @_ankahi @GeetaRaavi @pramodbansal64 @panditsripathak @raaga31280 @arunp131 @ahindu_ @ArunDeshpande20 @VineshGaba @chandanas @PCheppudira @BesuraTaansane @The_WittyLawyer @abs261 @ignisfatuus1110 @LibtardsKaDoc... | False | @sureshseshadri1 @prasad_perla @SortedEagle @shitul905 @shytigress @bs_hariharan3 @skbit1990 @Savitritvs @FireFly53647402 @chimnibai @Visalmama @Murali76133156 @nilesh_pat @swetamishraa @keeranl @neerangautam @RashmiShriJS @nair_hena @4mSingh @VivekRanjanSonb @kailashkaushik8 @umeshctyagi @Bhagirath1965 @Abhishe36743271 @Ravisha23624235 @Kaliyugamaya @ResurgentIndia @witnessconscio1 @_ankahi @GeetaRaavi @pramodbansal64 @panditsripathak @raaga31280 @arunp131 @ahindu_ @ArunDeshpande20 @VineshGaba @chandanas @PCheppudira @BesuraTaansane @The_WittyLawyer @abs261 @ignisfatuus1110 @LibtardsKaDoc... | perla hariharan3 pat hena ankahi WittyLawyer jyo Interview with Nasrine a Yazidi girl survivor of ISLAMIC State captivity YazidiPlight IslamicTerrorism 1 3 |
| 871 | 1 | @SharjeelUsmani @AfreenFatima136 Anywhere they go..they spread cancer of #terrorism #islamicterrorism https://t.co/6d2Sf0YMAO | True | @SharjeelUsmani @AfreenFatima136 Anywhere they go..they spread cancer of #terrorism #islamicterrorism | Anywhere they go they spread cancer of terrorism islamicterrorism |
| 872 | 1 | #Islamicterrorism kill humanity everywhere in the world #HinduUniteAgainstZehad | True | #Islamicterrorism kill humanity everywhere in the world #HinduUniteAgainstZehad | Islamicterrorism kill humanity everywhere in the world HinduUniteAgainstZehad |
| 873 | 1 | @SharjeelUsmani @AfreenFatima136 Just coz your religious slogan is war cry of #IslamicTerrorism you are doing information terrorism.. proving other religious slogans to be war cries.. so that you play victim card, escape freely etc. NO, ONLY YOUR RELIGIOUS SLOGAN IS A WAR CRY OF TERRORISTS GLOBALLY! | True | @SharjeelUsmani @AfreenFatima136 Just coz your religious slogan is war cry of #IslamicTerrorism you are doing information terrorism.. proving other religious slogans to be war cries.. so that you play victim card, escape freely etc. NO, ONLY YOUR RELIGIOUS SLOGAN IS A WAR CRY OF TERRORISTS GLOBALLY! | Just coz your religious slogan is war cry of IslamicTerrorism you are doing information terrorism proving other religious slogans to be war cries so that you play victim card escape freely etc NO ONLY YOUR RELIGIOUS SLOGAN IS A WAR CRY OF TERRORISTS GLOBALLY |
874 rows × 5 columns
from fastai.text.all import *df_text=df_final['Clean']df_text=df_text.to_frame()
df_text| Clean | |
|---|---|
| 0 | Group Selfie Apple AbhinavShukla Airdrop AEWRevolution2021 Airdrops BinanceSmartChain BSC ApplePay Gauahar JusticeForAgraKiBeti AuratMarch AjayDevgn HappyWomensDay HalaMadrid UnacademySachinFilm Trending Trendy KajalAggarwal InternationalWomensDay |
| 1 | News outlets and social media claimed that Mehtab had abducted the girl JusticeForAgraKiBeti was trending on Twitter However the police later clarified that the girl had plotted her own kidnapping with friends AltNewsFactCheck |
| 2 | News outlets and social media claimed that Mehtab had abducted the girl JusticeForAgraKiBeti was trending on Twitter However the police later clarified that the girl had plotted her own kidnapping with friends AltNewsFactCheck |
| 3 | News outlets and social media claimed that Mehtab had abducted the girl JusticeForAgraKiBeti was trending on Twitter However the police later clarified that the girl had plotted her own kidnapping with friends AltNewsFactCheck |
| 4 | News outlets and social media claimed that Mehtab had abducted the girl JusticeForAgraKiBeti was trending on Twitter However the police later clarified that the girl had plotted her own kidnapping with friends AltNewsFactCheck |
| ... | ... |
| 869 | Y all suck Yeh I said it Out if the measly 82 views not 1 person had the balls to retweet amp stand against HamasTerrorists ALL TERRORISM for that matter Sad IslamicTerrorism will continue to destroy lives Yours may be next Its happened before Remember911 NeverForget |
| 870 | perla hariharan3 pat hena ankahi WittyLawyer jyo Interview with Nasrine a Yazidi girl survivor of ISLAMIC State captivity YazidiPlight IslamicTerrorism 1 3 |
| 871 | Anywhere they go they spread cancer of terrorism islamicterrorism |
| 872 | Islamicterrorism kill humanity everywhere in the world HinduUniteAgainstZehad |
| 873 | Just coz your religious slogan is war cry of IslamicTerrorism you are doing information terrorism proving other religious slogans to be war cries so that you play victim card escape freely etc NO ONLY YOUR RELIGIOUS SLOGAN IS A WAR CRY OF TERRORISTS GLOBALLY |
874 rows × 1 columns
dls_lm = DataBlock(
blocks=TextBlock.from_df('Clean', is_lm=True),
get_x=ColReader('text'),
splitter=RandomSplitter(0.1)
# using only 10% of entire comments data for validation inorder to learn more
)dls_lm = dls_lm.dataloaders(df_text, bs=128,eq_len=80)/opt/conda/lib/python3.7/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray
return array(a, dtype, copy=False, order=order)dls_lm.show_batch(max_n=3)| text | text_ | |
|---|---|---|
| 0 | xxbos twochildpolicy populationcontrolbill populationpolitics xxmaj madhya xxmaj pradesh xxup bjp xxmaj leaders xxmaj demand xxup two xxup child xxmaj policy xxmaj like xxmaj that xxmaj in xxup up amp xxmaj assam national xxunk mp xxbos xxmaj xxunk xxup xxunk xxup xxunk xxunk xxunk xxmaj yes where are you xxmaj the muslims are xxunk non muslim xxunk everywhere xxmaj such as xxunk xxunk xxup isis alqaeda xxmaj taliban the international xxunk need to | twochildpolicy populationcontrolbill populationpolitics xxmaj madhya xxmaj pradesh xxup bjp xxmaj leaders xxmaj demand xxup two xxup child xxmaj policy xxmaj like xxmaj that xxmaj in xxup up amp xxmaj assam national xxunk mp xxbos xxmaj xxunk xxup xxunk xxup xxunk xxunk xxunk xxmaj yes where are you xxmaj the muslims are xxunk non muslim xxunk everywhere xxmaj such as xxunk xxunk xxup isis alqaeda xxmaj taliban the international xxunk need to stop |
| 1 | is xxmaj triple xxmaj talaq or triple divorce bill cum act in xxmaj india xxmaj see isrgrajan xxunk indianconstitution indianpolitics xxmaj islam marriage xxmaj muslims politicsinindia xxmaj relationship xxmaj talak xxmaj talaq tripletalaq xxbos xxmaj entrepreneurs and politicians please don t give us false hope and don t sell us fictional and untruthful all is well stories to sell your products or win the election tell people real scary stories that s | xxmaj triple xxmaj talaq or triple divorce bill cum act in xxmaj india xxmaj see isrgrajan xxunk indianconstitution indianpolitics xxmaj islam marriage xxmaj muslims politicsinindia xxmaj relationship xxmaj talak xxmaj talaq tripletalaq xxbos xxmaj entrepreneurs and politicians please don t give us false hope and don t sell us fictional and untruthful all is well stories to sell your products or win the election tell people real scary stories that s how |
| 2 | xxunk of xxup rss xxup bjp xxup modi xxup amitshah xxmaj pl don t xxunk xxmaj xxunk amp xxunk xxunk xxmaj common xxmaj man amp xxmaj working xxmaj xxunk r xxmaj angry xxunk xxmaj xxunk to xxmaj xxunk xxmaj xxunk in xxmaj everything i m xxmaj xxunk same happened when xxmaj demonetisation xxup gst xxup jk370 xxmaj tripletalaq xxmaj etc again they xxup won xxmaj xxunk xxmaj xxunk xxbos xxmaj the xxmaj | of xxup rss xxup bjp xxup modi xxup amitshah xxmaj pl don t xxunk xxmaj xxunk amp xxunk xxunk xxmaj common xxmaj man amp xxmaj working xxmaj xxunk r xxmaj angry xxunk xxmaj xxunk to xxmaj xxunk xxmaj xxunk in xxmaj everything i m xxmaj xxunk same happened when xxmaj demonetisation xxup gst xxup jk370 xxmaj tripletalaq xxmaj etc again they xxup won xxmaj xxunk xxmaj xxunk xxbos xxmaj the xxmaj modi |
learn = language_model_learner(
dls_lm, AWD_LSTM,drop_mult=0.3,
metrics=[accuracy, Perplexity()]).to_fp16()learn.lr_find()SuggestedLRs(lr_min=0.10964782238006592, lr_steep=0.25118863582611084)
learn.fit_one_cycle(1, 2e-2)| epoch | train_loss | valid_loss | accuracy | perplexity | time |
|---|---|---|---|---|---|
| 0 | 5.325597 | 5.058398 | 0.206321 | 157.338226 | 00:01 |
learn.save('1epoch')Path('models/1epoch.pth')learn.lr_find()SuggestedLRs(lr_min=0.03630780577659607, lr_steep=0.10000000149011612)
learn.unfreeze()
learn.fit_one_cycle(10)| epoch | train_loss | valid_loss | accuracy | perplexity | time |
|---|---|---|---|---|---|
| 0 | 4.806340 | 4.887314 | 0.208452 | 132.596985 | 00:01 |
| 1 | 4.679502 | 4.568364 | 0.230114 | 96.386261 | 00:01 |
| 2 | 4.542793 | 4.378533 | 0.246094 | 79.720985 | 00:02 |
| 3 | 4.410026 | 4.308868 | 0.237926 | 74.356262 | 00:01 |
| 4 | 4.292657 | 4.254110 | 0.254261 | 70.394165 | 00:01 |
| 5 | 4.185833 | 4.230815 | 0.257102 | 68.773285 | 00:01 |
| 6 | 4.090138 | 4.215041 | 0.258878 | 67.696922 | 00:01 |
| 7 | 4.008472 | 4.206219 | 0.260298 | 67.102356 | 00:01 |
| 8 | 3.938019 | 4.202753 | 0.262784 | 66.870148 | 00:02 |
| 9 | 3.879380 | 4.202420 | 0.263139 | 66.847893 | 00:01 |
learn.save_encoder('finetuned')TEXT = "i"
N_WORDS = 10
N_SENTENCES = 2
preds = [learn.predict(TEXT, N_WORDS, temperature=0.75)
for _ in range(N_SENTENCES)]print("\n".join(preds))i am a Muslim farmer and i am very proud
i am more concerned about criminal and criminal crimes indf_hate=df_final[df_final.Islamophobic==True]
df_neither=df_final[df_final.Islamophobic==False]rows3 = np.random.choice(df_neither.index.values, 200)
df_neither=df_neither.loc[rows3]li=[df_hate,df_neither]
df_new=pd.concat(li)
df_new| Annotation | Text | Islamophobic | Tweet | Clean | |
|---|---|---|---|---|---|
| 43 | 1 | @MeghUpdates To all saying Marzi se gyi that girl is just 17 yrs and that Jihadi Kidnapper is father of 6 kids and by marzi waale burqa pehna ke nhi leke jaate..@myogioffice Sir please help that girl #JusticeForAgraKiBeti | True | @MeghUpdates To all saying Marzi se gyi that girl is just 17 yrs and that Jihadi Kidnapper is father of 6 kids and by marzi waale burqa pehna ke nhi leke jaate..@myogioffice Sir please help that girl #JusticeForAgraKiBeti | To all saying Marzi se gyi that girl is just 17 yrs and that Jihadi Kidnapper is father of 6 kids and by marzi waale burqa pehna ke nhi leke jaate Sir please help that girl JusticeForAgraKiBeti |
| 56 | 1 | See how conversion mafia targeting only Hindu girls n distributing their propaganda books forcefully...🤬🤬\n#Protect_Hindu_Girls #JusticeForAgraKiBeti #boycotradicals #HindusLivesMatter https://t.co/Ds9CwPYd68 | True | See how conversion mafia targeting only Hindu girls n distributing their propaganda books forcefully...🤬🤬\n#Protect_Hindu_Girls #JusticeForAgraKiBeti #boycotradicals #HindusLivesMatter | See how conversion mafia targeting only Hindu girls n distributing their propaganda books forcefully Protect Hindu Girls JusticeForAgraKiBeti boycotradicals HindusLivesMatter |
| 65 | 1 | I am a Hindu I Support #Protect_Hindu_Girls\n#JusticeForAgraKiBeti https://t.co/Hd6YTdHcVv | True | I am a Hindu I Support #Protect_Hindu_Girls\n#JusticeForAgraKiBeti | I am a Hindu I Support Protect Hindu Girls JusticeForAgraKiBeti |
| 66 | 2 | I demand direct encounter should be done to Mehtab and his helpers if any wrong thing happen to girl Encounter should be done to whole family that,s it we don,t want one more Nikita tomar. #JusticeForAgraKiBeti https://t.co/OMWtUfGCzt | True | I demand direct encounter should be done to Mehtab and his helpers if any wrong thing happen to girl Encounter should be done to whole family that,s it we don,t want one more Nikita tomar. #JusticeForAgraKiBeti | I demand direct encounter should be done to Mehtab and his helpers if any wrong thing happen to girl Encounter should be done to whole family that s it we don t want one more Nikita tomar JusticeForAgraKiBeti |
| 73 | 2 | @Javedakhtarjadu Boss, you forgot to add the Rule of One Wife Per Man as well:P #TripleTalaq | True | @Javedakhtarjadu Boss, you forgot to add the Rule of One Wife Per Man as well:P #TripleTalaq | Boss you forgot to add the Rule of One Wife Per Man as well P TripleTalaq |
| ... | ... | ... | ... | ... | ... |
| 216 | 3 | In a big victory for Atiya Sabri, the #tripletalaq petitioner from UP, the court orders her husband to pay her an arrear of Rs 13.4 lakh and provide her monthly maintenance.\n\n#UttarPradesh\n\nhttps://t.co/8ayAFTj1yv | False | In a big victory for Atiya Sabri, the #tripletalaq petitioner from UP, the court orders her husband to pay her an arrear of Rs 13.4 lakh and provide her monthly maintenance.\n\n#UttarPradesh\n\n | In a big victory for Atiya Sabri the tripletalaq petitioner from UP the court orders her husband to pay her an arrear of Rs 13 4 lakh and provide her monthly maintenance UttarPradesh |
| 49 | 3 | @Nehasharmadubey @HinduDharma1 @VHPDigital @deeptisubba #JusticeForRinkuSharma \n#JusticeForAgraKiBeti \n#JusticeForSanghiNandu \n#BanPFI \n\nIt is Time to Show power of Hindu girls and Hindu Youth too....and set Fire Fire on \nTerrorist/Anti-national Liberals.\nLiberals get Burned.\n\n#Protect_Hindu_Girls https://t.co/5SUaVKITNQ | False | @Nehasharmadubey @HinduDharma1 @VHPDigital @deeptisubba #JusticeForRinkuSharma \n#JusticeForAgraKiBeti \n#JusticeForSanghiNandu \n#BanPFI \n\nIt is Time to Show power of Hindu girls and Hindu Youth too....and set Fire Fire on \nTerrorist/Anti-national Liberals.\nLiberals get Burned.\n\n#Protect_Hindu_Girls | JusticeForRinkuSharma JusticeForAgraKiBeti JusticeForSanghiNandu BanPFI It is Time to Show power of Hindu girls and Hindu Youth too and set Fire Fire on Terrorist Anti national Liberals Liberals get Burned Protect Hindu Girls |
| 35 | 3 | Whatever happened to Ayesha went horribly wrong 😔 And I strongly condemn it.\nBut I also strongly condemn Ayesha's Action.\n#ayeshaarifkhan \n#JusticeForAgraKiBeti | False | Whatever happened to Ayesha went horribly wrong 😔 And I strongly condemn it.\nBut I also strongly condemn Ayesha's Action.\n#ayeshaarifkhan \n#JusticeForAgraKiBeti | Whatever happened to Ayesha went horribly wrong And I strongly condemn it But I also strongly condemn Ayesha s Action ayeshaarifkhan JusticeForAgraKiBeti |
| 665 | 3 | @RifatJawaid Leave Hindutva morons aside, mind telling which religion's terrorists killed Danish?\n#DanishSiddique \n#Taliban \n#Afghanistan \n#IslamicTerrorism\n#RadicalIslamIsTheProblem | False | @RifatJawaid Leave Hindutva morons aside, mind telling which religion's terrorists killed Danish?\n#DanishSiddique \n#Taliban \n#Afghanistan \n#IslamicTerrorism\n#RadicalIslamIsTheProblem | Leave Hindutva morons aside mind telling which religion s terrorists killed Danish DanishSiddique Taliban Afghanistan IslamicTerrorism RadicalIslamIsTheProblem |
| 86 | 3 | @khanumarfa And many such where women have told press that they ate forced to go and participate in these Dharnas \nIs this u r definition of #womenempowerment \n\n#halala #tripletalaq #burkha - when r u writing about this https://t.co/z2wuJUBI3L | False | @khanumarfa And many such where women have told press that they ate forced to go and participate in these Dharnas \nIs this u r definition of #womenempowerment \n\n#halala #tripletalaq #burkha - when r u writing about this | And many such where women have told press that they ate forced to go and participate in these Dharnas Is this u r definition of womenempowerment halala tripletalaq burkha when r u writing about this |
385 rows × 5 columns
df_new.drop(['Annotation','Text','Tweet'],axis=1,inplace=True)df_new.reset_index(drop=True,inplace=True)
df_new| Islamophobic | Clean | |
|---|---|---|
| 0 | True | To all saying Marzi se gyi that girl is just 17 yrs and that Jihadi Kidnapper is father of 6 kids and by marzi waale burqa pehna ke nhi leke jaate Sir please help that girl JusticeForAgraKiBeti |
| 1 | True | See how conversion mafia targeting only Hindu girls n distributing their propaganda books forcefully Protect Hindu Girls JusticeForAgraKiBeti boycotradicals HindusLivesMatter |
| 2 | True | I am a Hindu I Support Protect Hindu Girls JusticeForAgraKiBeti |
| 3 | True | I demand direct encounter should be done to Mehtab and his helpers if any wrong thing happen to girl Encounter should be done to whole family that s it we don t want one more Nikita tomar JusticeForAgraKiBeti |
| 4 | True | Boss you forgot to add the Rule of One Wife Per Man as well P TripleTalaq |
| ... | ... | ... |
| 380 | False | In a big victory for Atiya Sabri the tripletalaq petitioner from UP the court orders her husband to pay her an arrear of Rs 13 4 lakh and provide her monthly maintenance UttarPradesh |
| 381 | False | JusticeForRinkuSharma JusticeForAgraKiBeti JusticeForSanghiNandu BanPFI It is Time to Show power of Hindu girls and Hindu Youth too and set Fire Fire on Terrorist Anti national Liberals Liberals get Burned Protect Hindu Girls |
| 382 | False | Whatever happened to Ayesha went horribly wrong And I strongly condemn it But I also strongly condemn Ayesha s Action ayeshaarifkhan JusticeForAgraKiBeti |
| 383 | False | Leave Hindutva morons aside mind telling which religion s terrorists killed Danish DanishSiddique Taliban Afghanistan IslamicTerrorism RadicalIslamIsTheProblem |
| 384 | False | And many such where women have told press that they ate forced to go and participate in these Dharnas Is this u r definition of womenempowerment halala tripletalaq burkha when r u writing about this |
385 rows × 2 columns
df_new = df_new.sample(frac=1).reset_index(drop=True)dls = TextDataLoaders.from_df(df_new, text_col='Clean', label_col='Islamophobic',valid_pct=0.1,text_vocab=dls_lm.vocab)/opt/conda/lib/python3.7/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray
return array(a, dtype, copy=False, order=order)dls.show_batch(max_n=3)| text | category | |
|---|---|---|
| 0 | xxbos 4 s a s xxunk xxmaj xxunk xxunk xxunk xxunk xxunk xxunk xxunk 3 13 xxunk xxunk xxunk ayeshaarifkhand live suicide xxmaj its due to cruel la ws of tripletalaq xxmaj arif did not want to give talaq amp go to jail xxmaj she did not want to be xxmaj yashodaben living a painfull long life xxmaj she could not get marry unless xxmaj arif would not give her xxmaj talaq xxmaj muslims shud shun hindusdowrysystem | False |
| 1 | xxbos 4 s a s xxunk xxmaj xxunk xxunk xxunk xxunk xxunk xxunk xxunk 3 13 xxunk xxunk xxunk ayeshaarifkhand live suicide xxmaj its due to cruel la ws of tripletalaq xxmaj arif did not want to give talaq amp go to jail xxmaj she did not want to be xxmaj yashodaben living a painfull long life xxmaj she could not get marry unless xxmaj arif would not give her xxmaj talaq xxmaj muslims shud shun hindusdowrysystem | False |
| 2 | xxbos islamicterrorism xxunk xxmaj so xxmaj many times on xxmaj past xxmaj clearly xxmaj exposed xxmaj false xxmaj propaganda xxmaj spreading xxmaj politicians xxmaj like xxmaj secularism xxmaj bhai xxmaj chara xxmaj xxunk xxmaj xxunk xxmaj xxunk xxmaj all xxmaj religion xxmaj are xxmaj equal xxmaj dear xxmaj hindus xxmaj wake xxup up xxmaj call xxmaj for xxmaj you xxmaj leave xxmaj secularism amp xxmaj bhai xxmaj chara xxmaj as xxmaj early xxmaj as xxmaj xxunk | False |
learn = text_classifier_learner(dls,AWD_LSTM,metrics=accuracy,
drop_mult=0.5).to_fp16()learn = learn.load_encoder('finetuned')learn.lr_find()SuggestedLRs(lr_min=0.06309573650360108, lr_steep=0.033113110810518265)
learn.fit_one_cycle(1,1e-2)| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.826446 | 0.615068 | 0.763158 | 00:01 |
learn.freeze_to(-2)
# learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2))
#learn.lr_find()learn.lr_find()SuggestedLRs(lr_min=0.002290867641568184, lr_steep=0.015848932787775993)
learn.fit_one_cycle(1)| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.590375 | 0.598704 | 0.763158 | 00:00 |
learn.freeze_to(-3)
# learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3))
learn.lr_find()SuggestedLRs(lr_min=0.0009120108559727668, lr_steep=0.002511886414140463)
learn.fit_one_cycle(1)| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.532034 | 0.577508 | 0.736842 | 00:01 |
learn.unfreeze()
# learn.fit_one_cycle(5, slice(1e-3/(2.6**4),1e-3))
# learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3))
learn.lr_find()SuggestedLRs(lr_min=0.0009120108559727668, lr_steep=0.004365158267319202)
learn.fit_one_cycle(5)| epoch | train_loss | valid_loss | accuracy | time |
|---|---|---|---|---|
| 0 | 0.440877 | 0.593707 | 0.710526 | 00:01 |
| 1 | 0.419042 | 0.582485 | 0.657895 | 00:01 |
| 2 | 0.358577 | 0.470297 | 0.789474 | 00:01 |
| 3 | 0.316723 | 0.488469 | 0.815789 | 00:01 |
| 4 | 0.278898 | 0.489707 | 0.815789 | 00:01 |
interp = ClassificationInterpretation.from_learner(learn)
interp.plot_confusion_matrix(figsize=(12,12), dpi=60)
interp.print_classification_report() precision recall f1-score support
False 0.94 0.73 0.82 22
True 0.71 0.94 0.81 16
accuracy 0.82 38
macro avg 0.83 0.83 0.82 38
weighted avg 0.85 0.82 0.82 38