Extract the language you need from the multilingual data set and save it as CSV

If there are multiple languages in the dataset , How to recognize each language ？ What I borrow here is langdetect Package function , Don't talk much , Look at the code .

# -*- coding: utf-8 -*-
#  Support detection 55 Languages ： af, ar, bg, bn, ca, cs, cy, da, de, el, en, es, et, fa, fi, fr, gu, he, hi, hr, hu, id, it, ja, kn, ko, lt, lv, mk, ml, mr, ne, nl, no, pa, pl, pt, ro, ru, sk, sl, so, sq, sv, sw, ta, te, th, tl, tr, uk, ur, vi, zh-cn, zh-tw

# str = 'Otec matka syn.'

from langdetect import detect
from langdetect import detect_langs

#  When the text is too short or fuzzy , The result of judgment will be uncertain ;
#  If you want to make the result unique , Add the following two lines ：
from langdetect import DetectorFactory
DetectorFactory.seed = 0






if __name__ == '__main__':
    
    #  This paragraph str It's after cleaning the web page , Take its text directly 
    str = ' Chinese corpus '
    #  Judge the type of language 
    print(detect(str))

    #  probability 
    print(detect_langs(str))



def getLangs(str) :
    return detect(str)

Dataset format

3,In Ordnung other,"Die Hülle ist schön, jedoch nicht ganz mein Geschmack."
3, Try the battery without しでした. drugstore,すでにこのタイプの Compensator をもってらっしゃる Fang は、 Battery も Hand yuan にあるのでしょうが、 first めて Starting with した body としては、 Try the battery がついてなかったことは、ちょっとがっかりさせられました. today までニコンイヤーファッションなどの Ear point tonic device を Buy った Age には、たいてい Try the battery が1 One ついてましたから. Mender body の send い The winner などは、これからぼちぼちみていきたいと thinking います.
5,Super kit kitchen,"J'ai changé toutes les pièces facilement, c'est top merci"
1,toujours pas recu home,toujour s pas reçu ou est mon colis

The method of extracting language written in this format , Other formats can be fine tuned

'''  Extract the corresponding language from the data set  '''
import csv
import pandas as pd
import os
from extractLanguage import getLangs
# data_path = "NAO/NAO-WS/rnn/data/amazon_test_data"
data_path = "data" #  The directory where the dataset is located 
need_langs = ['zh_cn', 'en', 'de'] #  Required language 
file_names = ['eval.csv', 'test.csv', 'train.csv'] #  The name of the file under the dataset 
for need_lang in need_langs:
    store_path = data_path + "/" + need_lang #  The directory where the generated files are saved 
    if not os.path.exists(store_path):
        os.makedirs(store_path)
    for name in file_names:
        df = pd.read_csv(data_path + "/" + name, sep=",", header=None)
        data_lang = []
        for i in df.itertuples():
            row_data = list(i)
            text_data = str(row_data[2]) + str(row_data[3])
            score = row_data[1]
            lang = getLangs(text_data)
            if lang == need_lang:
                temp = [row_data[1], row_data[2], row_data[3]]
                data_lang.append(temp)
            # print(list(i))
        df_lang = pd.DataFrame(data_lang)
        df_lang.to_csv(store_path + "/" + name, index=False, header=None)

If you have other questions, please contact me