MCPcopy Create free account
hub / github.com/activeloopai/deeplake / do_inserts

Function do_inserts

postgres/tests/py_tests/test_concurrent_insert_index.py:61–146  ·  view source on GitHub ↗
()

Source from the content-addressed store, hash-verified

59 Runs in separate process to avoid blocking.
60 """
61 async def do_inserts():
62 user = os.environ.get("USER", "postgres")
63 conn = await asyncpg.connect(
64 database="postgres",
65 user=user,
66 host="localhost",
67 statement_cache_size=0
68 )
69
70 try:
71 print(f"Insert worker: Starting {insert_count} inserts in batches of {batch_size}...")
72
73 completed = 0
74 batch_num = 0
75
76 while completed < insert_count:
77 current_batch_size = min(batch_size, insert_count - completed)
78 batch_num += 1
79
80 # Single insert for first iteration
81 if completed == 0:
82 await conn.execute("""
83 INSERT INTO chunk_text_image (id, text, image, metadata, chunk_index, file_id, created_at)
84 VALUES (
85 'chunk_12345',
86 'This is the text content of the chunk',
87 '\\xDEADBEEF'::bytea,
88 '{"key": "value", "tags": ["tag1", "tag2"]}'::jsonb,
89 1,
90 '550e8400-e29b-41d4-a716-446655440000'::uuid,
91 1700000000
92 )
93 """)
94 completed += 1
95 print(f"Insert worker: Completed 1 insert")
96 continue
97
98 # Batch inserts
99 values_list = []
100 for i in range(current_batch_size):
101 idx = completed + i
102 chunk_id = f'chunk_{idx:09d}'
103 text = f'Chunk text content number {idx}'
104
105 if idx % 5 == 0:
106 image = 'NULL'
107 elif idx % 3 == 0:
108 image = "'\\x89504E47'::bytea"
109 else:
110 image = "'\\xDEADBEEF'::bytea"
111
112 metadata = f'{{"index": {idx}, "batch": {batch_num}}}'
113 chunk_index = idx
114 file_id = '550e8400-e29b-41d4-a716-446655440000' if idx % 2 == 0 else '660e8400-e29b-41d4-a716-446655440001'
115 created_at = 1700000000 + idx
116
117 values_list.append(
118 f"('{chunk_id}', '{text}', {image}, '{metadata}'::jsonb, "

Callers 1

insert_worker_processFunction · 0.85

Calls 3

connectMethod · 0.80
appendMethod · 0.80
getMethod · 0.45

Tested by

no test coverage detected