Dask Bag の概要

Pythonで学ぶDaskによる並列プログラミング

James Fulton

Climate Informatics Researcher

非構造化データとは？

# 非構造化テキストデータ
string_list = [
    "Really good service ...",
    "This is the second time we've stayed ...", 
    "Great older hotel. My husband took ...",
    ...]

# 半構造化の辞書データ
dict_list = [
    {"name": "Beth", "employment": [{"role": "manager", "start_date": ...}, ...]},
    {"name": "Omar", "employment": [{"role": "analyst", "start_date": ...}, ...]},
    {"name": "Fang", "employment": [{"role": "engineer", "start_date": ...}, ...]},
    ...]

Dask Bag

import dask.bag as db


# リストから Dask Bag を作成
bag_example = db.from_sequence(string_list, npartitions=5)

print(bag_example)

dask.bag<from_sequence, npartitions=5>

# Bag から要素を1件表示
print(bag_example.take(1))

('Really good service ...',)

Dask Bag

import dask.bag as db

# リストから Dask Bag を作成
bag_example = db.from_sequence(string_list, npartitions=5)
print(bag_example)

dask.bag<from_sequence, npartitions=5>

# 要素を2件表示
print(bag_example.take(2))

('Really good service ...', 'This is the second time we've stayed ...'',)

要素数

number_of_elements = bag_example.count()
print(number_of_elements)

<dask.bag.core.Item at ...>

print(number_of_elements.compute())

テキストデータの読み込み

import glob

filenames = glob.glob('data/*.txt')

print(filenames)

["data/file_0.txt", "data/file_1.txt", "data/file_2.txt"]

text_data_bag = db.read_text(filenames)

text_data_bag = db.read_text('data/*.txt')

print(text_data_bag)

dask.bag<bag-from-delayed, npartitions=3>

文字列操作

text_data_bag = db.read_text('data/*.txt')

print(text_data_bag.take(1))

('Really good service ...',)

# 文字列を小文字に変換
print(text_data_bag.str.lower().take(1))

('really good service ...',)

文字列操作

# すべての 'good' を 'great' に置換
print(text_data_bag.str.replace('good', 'great').take(1))

('Really great service ...',)

# Bag の先頭3要素で 'great' が出現する回数
print(text_data_bag.str.count('great').take(3))

(0,1,5,)

Passons à la pratique !

Pythonで学ぶDaskによる並列プログラミング