/home/brain/Projects/upstream/libsolv/src/repodata.c

Bug Summary

File:	src/repodata.c
Warning:	line 658, column 3 Value stored to 'dp' is never read

Annotated Source Code

1	/*
2	* Copyright (c) 2007, Novell Inc.
3	*
4	* This program is licensed under the BSD license, read LICENSE.BSD
5	* for further information
6	*/
7
8	/*
9	* repodata.c
10	*
11	* Manage data coming from one repository
12	*
13	* a repository can contain multiple repodata entries, consisting of
14	* different sets of keys and different sets of solvables
15	*/
16
17	#define _GNU_SOURCE
18	#include <string.h>
19	#include <fnmatch.h>
20
21	#include <stdio.h>
22	#include <stdlib.h>
23	#include <unistd.h>
24	#include <assert.h>
25	#include <regex.h>
26
27	#include "repo.h"
28	#include "pool.h"
29	#include "poolid_private.h"
30	#include "util.h"
31	#include "hash.h"
32	#include "chksum.h"
33
34	#include "repopack.h"
35	#include "repopage.h"
36
37	#define REPODATA_BLOCK255 255
38
39	static unsigned char data_skip_key(Repodata data, unsigned char dp, Repokey key);
40
41	void
42	repodata_initdata(Repodata data, Repo repo, int localpool)
43	{
44	memset(data, 0, sizeof (*data));
45	data->repodataid = data - repo->repodata;
46	data->repo = repo;
47	data->localpool = localpool;
48	if (localpool)
49	stringpool_init_empty(&data->spool);
50	/* dirpool_init(&data->dirpool); just zeros out again */
51	data->keys = solv_calloc(1, sizeof(Repokey));
52	data->nkeys = 1;
53	data->schemata = solv_calloc(1, sizeof(Id));
54	data->schemadata = solv_calloc(1, sizeof(Id));
55	data->nschemata = 1;
56	data->schemadatalen = 1;
57	repopagestore_init(&data->store);
58	}
59
60	void
61	repodata_freedata(Repodata *data)
62	{
63	int i;
64
65	solv_free(data->keys);
66
67	solv_free(data->schemata);
68	solv_free(data->schemadata);
69	solv_free(data->schematahash);
70
71	stringpool_free(&data->spool);
72	dirpool_free(&data->dirpool);
73
74	solv_free(data->mainschemaoffsets);
75	solv_free(data->incoredata);
76	solv_free(data->incoreoffset);
77	solv_free(data->verticaloffset);
78
79	repopagestore_free(&data->store);
80
81	solv_free(data->vincore);
82
83	if (data->attrs)
84	for (i = 0; i < data->end - data->start; i++)
85	solv_free(data->attrs[i]);
86	solv_free(data->attrs);
87	if (data->xattrs)
88	for (i = 0; i < data->nxattrs; i++)
89	solv_free(data->xattrs[i]);
90	solv_free(data->xattrs);
91
92	solv_free(data->attrdata);
93	solv_free(data->attriddata);
94	solv_free(data->attrnum64data);
95
96	solv_free(data->dircache);
97	}
98
99	void
100	repodata_free(Repodata *data)
101	{
102	Repo *repo = data->repo;
103	int i = data - repo->repodata;
104	if (i == 0)
105	return;
106	repodata_freedata(data);
107	if (i < repo->nrepodata - 1)
108	{
109	/* whoa! this changes the repodataids! */
110	memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
111	for (; i < repo->nrepodata - 1; i++)
112	repo->repodata[i].repodataid = i;
113	}
114	repo->nrepodata--;
115	if (repo->nrepodata == 1)
116	{
117	repo->repodata = solv_free(repo->repodata);
118	repo->nrepodata = 0;
119	}
120	}
121
122	void
123	repodata_empty(Repodata *data, int localpool)
124	{
125	void (loadcallback)(Repodata ) = data->loadcallback;
126	int state = data->state;
127	repodata_freedata(data);
128	repodata_initdata(data, data->repo, localpool);
129	data->state = state;
130	data->loadcallback = loadcallback;
131	}
132
133
134	/***************************************************************
135	* key pool management
136	*/
137
138	/* this is not so time critical that we need a hash, so we do a simple
139	* linear search */
140	Id
141	repodata_key2id(Repodata data, Repokey key, int create)
142	{
143	Id keyid;
144
145	for (keyid = 1; keyid < data->nkeys; keyid++)
146	if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
147	{
148	if ((key->type == REPOKEY_TYPE_CONSTANT \|\| key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
149	continue;
150	break;
151	}
152	if (keyid == data->nkeys)
153	{
154	if (!create)
155	return 0;
156	/* allocate new key */
157	data->keys = solv_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
158	data->keys[data->nkeys++] = *key;
159	if (data->verticaloffset)
160	{
161	data->verticaloffset = solv_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
162	data->verticaloffset[data->nkeys - 1] = 0;
163	}
164	data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] \|= 1 << (key->name & 7);
165	}
166	return keyid;
167	}
168
169
170	/***************************************************************
171	* schema pool management
172	*/
173
174	#define SCHEMATA_BLOCK31 31
175	#define SCHEMATADATA_BLOCK255 255
176
177	Id
178	repodata_schema2id(Repodata data, Id schema, int create)
179	{
180	int h, len, i;
181	Id *sp, cid;
182	Id *schematahash;
183
184	if (!*schema)
185	return 0; /* XXX: allow empty schema? */
186	if ((schematahash = data->schematahash) == 0)
187	{
188	data->schematahash = schematahash = solv_calloc(256, sizeof(Id));
189	for (i = 1; i < data->nschemata; i++)
190	{
191	for (sp = data->schemadata + data->schemata[i], h = 0; *sp;)
192	h = h * 7 + *sp++;
193	h &= 255;
194	schematahash[h] = i;
195	}
196	data->schemadata = solv_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK255);
197	data->schemata = solv_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK31);
198	}
199
200	for (sp = schema, len = 0, h = 0; *sp; len++)
201	h = h * 7 + *sp++;
202	h &= 255;
203	len++;
204
205	cid = schematahash[h];
206	if (cid)
207	{
208	if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
209	return cid;
210	/* cache conflict, do a slow search */
211	for (cid = 1; cid < data->nschemata; cid++)
212	if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
213	return cid;
214	}
215	/* a new one */
216	if (!create)
217	return 0;
218	data->schemadata = solv_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK255);
219	data->schemata = solv_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK31);
220	/* add schema */
221	memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
222	data->schemata[data->nschemata] = data->schemadatalen;
223	data->schemadatalen += len;
224	schematahash[h] = data->nschemata;
225	#if 0
226	fprintf(stderrstderr, "schema2id: new schema\n");
227	#endif
228	return data->nschemata++;
229	}
230
231	void
232	repodata_free_schemahash(Repodata *data)
233	{
234	data->schematahash = solv_free(data->schematahash);
235	/* shrink arrays */
236	data->schemata = solv_realloc2(data->schemata, data->nschemata, sizeof(Id));
237	data->schemadata = solv_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
238	}
239
240
241	/***************************************************************
242	* dir pool management
243	*/
244
245	#ifndef HAVE_STRCHRNUL1
246	static inline const char strchrnul(const char str, char x)
247	{
248	const char *p = strchr(str, x);
249	return p ? p : str + strlen(str);
250	}
251	#endif
252
253	#define DIRCACHE_SIZE41 41 /* < 1k */
254
255	#ifdef DIRCACHE_SIZE41
256	struct dircache {
257	Id ids[DIRCACHE_SIZE41];
258	char str[(DIRCACHE_SIZE41 * (DIRCACHE_SIZE41 - 1)) / 2];
259	};
260	#endif
261
262	Id
263	repodata_str2dir(Repodata data, const char dir, int create)
264	{
265	Id id, parent;
266	#ifdef DIRCACHE_SIZE41
267	const char *dirs;
268	#endif
269	const char *dire;
270
271	if (!*dir)
272	return data->dirpool.ndirs ? 0 : dirpool_add_dir(&data->dirpool, 0, 0, create);
273	while (*dir == '/' && dir[1] == '/')
274	dir++;
275	if (*dir == '/' && !dir[1])
276	return data->dirpool.ndirs ? 1 : dirpool_add_dir(&data->dirpool, 0, 1, create);
277	parent = 0;
278	#ifdef DIRCACHE_SIZE41
279	dirs = dir;
280	if (data->dircache)
281	{
282	int l;
283	struct dircache *dircache = data->dircache;
284	l = strlen(dir);
285	while (l > 0)
286	{
287	if (l < DIRCACHE_SIZE41 && dircache->ids[l] && !memcmp(dircache->str + l * (l - 1) / 2, dir, l))
288	{
289	parent = dircache->ids[l];
290	dir += l;
291	if (!*dir)
292	return parent;
293	while (*dir == '/')
294	dir++;
295	break;
296	}
297	while (--l)
298	if (dir[l] == '/')
299	break;
300	}
301	}
302	#endif
303	while (*dir)
304	{
305	dire = strchrnul(dir, '/');
306	if (data->localpool)
307	id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
308	else
309	id = pool_strn2id(data->repo->pool, dir, dire - dir, create);
310	if (!id)
311	return 0;
312	parent = dirpool_add_dir(&data->dirpool, parent, id, create);
313	if (!parent)
314	return 0;
315	#ifdef DIRCACHE_SIZE41
316	if (!data->dircache)
317	data->dircache = solv_calloc(1, sizeof(struct dircache));
318	if (data->dircache)
319	{
320	int l = dire - dirs;
321	if (l < DIRCACHE_SIZE41)
322	{
323	data->dircache->ids[l] = parent;
324	memcpy(data->dircache->str + l * (l - 1) / 2, dirs, l);
325	}
326	}
327	#endif
328	if (!*dire)
329	break;
330	dir = dire + 1;
331	while (*dir == '/')
332	dir++;
333	}
334	return parent;
335	}
336
337	void
338	repodata_free_dircache(Repodata *data)
339	{
340	data->dircache = solv_free(data->dircache);
341	}
342
343	const char *
344	repodata_dir2str(Repodata data, Id did, const char suf)
345	{
346	Pool *pool = data->repo->pool;
347	int l = 0;
348	Id parent, comp;
349	const char *comps;
350	char *p;
351
352	if (!did)
353	return suf ? suf : "";
354	if (did == 1 && !suf)
355	return "/";
356	parent = did;
357	while (parent)
358	{
359	comp = dirpool_compid(&data->dirpool, parent);
360	comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
361	l += strlen(comps);
362	parent = dirpool_parent(&data->dirpool, parent);
363	if (parent)
364	l++;
365	}
366	if (suf)
367	l += strlen(suf) + 1;
368	p = pool_alloctmpspace(pool, l + 1) + l;
369	*p = 0;
370	if (suf)
371	{
372	p -= strlen(suf);
373	strcpy(p, suf);
374	*--p = '/';
375	}
376	parent = did;
377	while (parent)
378	{
379	comp = dirpool_compid(&data->dirpool, parent);
380	comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
381	l = strlen(comps);
382	p -= l;
383	strncpy(p, comps, l);
384	parent = dirpool_parent(&data->dirpool, parent);
385	if (parent)
386	*--p = '/';
387	}
388	return p;
389	}
390
391
392	/***************************************************************
393	* data management
394	*/
395
396	static inline unsigned char *
397	data_skip_schema(Repodata data, unsigned char dp, Id schema)
398	{
399	Id *keyp = data->schemadata + data->schemata[schema];
400	for (; *keyp; keyp++)
401	dp = data_skip_key(data, dp, data->keys + *keyp);
402	return dp;
403	}
404
405	static unsigned char *
406	data_skip_key(Repodata data, unsigned char dp, Repokey *key)
407	{
408	int nentries, schema;
409	switch(key->type)
410	{
411	case REPOKEY_TYPE_FIXARRAY:
412	dp = data_read_id(dp, &nentries);
413	if (!nentries)
414	return dp;
415	dp = data_read_id(dp, &schema);
416	while (nentries--)
417	dp = data_skip_schema(data, dp, schema);
418	return dp;
419	case REPOKEY_TYPE_FLEXARRAY:
420	dp = data_read_id(dp, &nentries);
421	while (nentries--)
422	{
423	dp = data_read_id(dp, &schema);
424	dp = data_skip_schema(data, dp, schema);
425	}
426	return dp;
427	default:
428	if (key->storage == KEY_STORAGE_INCORE2)
429	dp = data_skip(dp, key->type);
430	else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET3)
431	{
432	dp = data_skip(dp, REPOKEY_TYPE_ID);
433	dp = data_skip(dp, REPOKEY_TYPE_ID);
434	}
435	return dp;
436	}
437	}
438
439	static unsigned char *
440	forward_to_key(Repodata data, Id keyid, Id keyp, unsigned char *dp)
441	{
442	Id k;
443
444	if (!keyid)
445	return 0;
446	if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
447	{
448	int i;
449	for (i = 0; (k = *keyp++) != 0; i++)
450	if (k == keyid)
451	return data->incoredata + data->mainschemaoffsets[i];
452	return 0;
453	}
454	while ((k = *keyp++) != 0)
455	{
456	if (k == keyid)
457	return dp;
458	if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET3)
459	{
460	dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
461	dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
462	continue;
463	}
464	if (data->keys[k].storage != KEY_STORAGE_INCORE2)
465	continue;
466	dp = data_skip_key(data, dp, data->keys + k);
467	}
468	return 0;
469	}
470
471	static unsigned char *
472	get_vertical_data(Repodata data, Repokey key, Id off, Id len)
473	{
474	unsigned char *dp;
475	if (len <= 0)
476	return 0;
477	if (off >= data->lastverticaloffset)
478	{
479	off -= data->lastverticaloffset;
480	if ((unsigned int)off + len > data->vincorelen)
481	return 0;
482	return data->vincore + off;
483	}
484	if ((unsigned int)off + len > key->size)
485	return 0;
486	/* we now have the offset, go into vertical */
487	off += data->verticaloffset[key - data->keys];
488	/* fprintf(stderr, "key %d page %d\n", key->name, off / REPOPAGE_BLOBSIZE); */
489	dp = repopagestore_load_page_range(&data->store, off / REPOPAGE_BLOBSIZE(1 << 15), (off + len - 1) / REPOPAGE_BLOBSIZE(1 << 15));
490	data->storestate++;
491	if (dp)
492	dp += off % REPOPAGE_BLOBSIZE(1 << 15);
493	return dp;
494	}
495
496	static inline unsigned char *
497	get_data(Repodata data, Repokey key, unsigned char **dpp, int advance)
498	{
499	unsigned char dp = dpp;
500
501	if (!dp)
502	return 0;
503	if (key->storage == KEY_STORAGE_INCORE2)
504	{
505	if (advance)
506	*dpp = data_skip_key(data, dp, key);
507	return dp;
508	}
509	else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET3)
510	{
511	Id off, len;
512	dp = data_read_id(dp, &off);
513	dp = data_read_id(dp, &len);
514	if (advance)
515	*dpp = dp;
516	return get_vertical_data(data, key, off, len);
517	}
518	return 0;
519	}
520
521	static int
522	load_repodata(Repodata *data)
523	{
524	if (data->loadcallback)
525	{
526	data->loadcallback(data);
527	if (data->state == REPODATA_AVAILABLE0)
528	return 1;
529	}
530	data->state = REPODATA_ERROR2;
531	return 0;
532	}
533
534	static inline int
535	maybe_load_repodata(Repodata *data, Id keyname)
536	{
537	if (keyname && !repodata_precheck_keyname(data, keyname))
538	return 0; /* do not bother... */
539	switch(data->state)
540	{
541	case REPODATA_STUB1:
542	if (keyname)
543	{
544	int i;
545	for (i = 1; i < data->nkeys; i++)
546	if (keyname == data->keys[i].name)
547	break;
548	if (i == data->nkeys)
549	return 0;
550	}
551	return load_repodata(data);
552	case REPODATA_ERROR2:
553	return 0;
554	case REPODATA_AVAILABLE0:
555	case REPODATA_LOADING4:
556	return 1;
557	default:
558	data->state = REPODATA_ERROR2;
559	return 0;
560	}
561	}
562
563	static inline unsigned char *
564	solvid2data(Repodata data, Id solvid, Id schemap)
565	{
566	unsigned char *dp = data->incoredata;
567	if (!dp)
568	return 0;
569	if (solvid == SOLVID_META-1)
570	dp += 1; /* offset of "meta" solvable */
571	else if (solvid == SOLVID_POS-2)
572	{
573	Pool *pool = data->repo->pool;
574	if (data->repo != pool->pos.repo)
575	return 0;
576	if (data != data->repo->repodata + pool->pos.repodataid)
577	return 0;
578	dp += pool->pos.dp;
579	if (pool->pos.dp != 1)
580	{
581	*schemap = pool->pos.schema;
582	return dp;
583	}
584	}
585	else
586	{
587	if (solvid < data->start \|\| solvid >= data->end)
588	return 0;
589	dp += data->incoreoffset[solvid - data->start];
590	}
591	return data_read_id(dp, schemap);
592	}
593
594	/************************************************************************
595	* data lookup
596	*/
597
598	static unsigned char *
599	find_key_data(Repodata data, Id solvid, Id keyname, Repokey *keypp)
600	{
601	unsigned char *dp;
602	Id schema, keyp, kp;
603	Repokey *key;
604
605	if (!maybe_load_repodata(data, keyname))
606	return 0;
607	dp = solvid2data(data, solvid, &schema);
608	if (!dp)
609	return 0;
610	keyp = data->schemadata + data->schemata[schema];
611	for (kp = keyp; *kp; kp++)
612	if (data->keys[*kp].name == keyname)
613	break;
614	if (!*kp)
615	return 0;
616	keypp = key = data->keys + kp;
617	if (key->type == REPOKEY_TYPE_DELETED)
618	return 0;
619	if (key->type == REPOKEY_TYPE_VOID \|\| key->type == REPOKEY_TYPE_CONSTANT \|\| key->type == REPOKEY_TYPE_CONSTANTID)
620	return dp; /* no need to forward... */
621	if (key->storage != KEY_STORAGE_INCORE2 && key->storage != KEY_STORAGE_VERTICAL_OFFSET3)
622	return 0; /* get_data will not work, no need to forward */
623	dp = forward_to_key(data, *kp, keyp, dp);
624	if (!dp)
625	return 0;
626	return get_data(data, key, &dp, 0);
627	}
628
629	Id
630	repodata_lookup_type(Repodata *data, Id solvid, Id keyname)
631	{
632	Id schema, keyp, kp;
633	if (!maybe_load_repodata(data, keyname))
634	return 0;
635	if (!solvid2data(data, solvid, &schema))
636	return 0;
637	keyp = data->schemadata + data->schemata[schema];
638	for (kp = keyp; *kp; kp++)
639	if (data->keys[*kp].name == keyname)
640	return data->keys[*kp].type;
641	return 0;
642	}
643
644	Id
645	repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
646	{
647	unsigned char *dp;
648	Repokey *key;
649	Id id;
650
651	dp = find_key_data(data, solvid, keyname, &key);
652	if (!dp)
653	return 0;
654	if (key->type == REPOKEY_TYPE_CONSTANTID)
655	return key->size;
656	if (key->type != REPOKEY_TYPE_ID)
657	return 0;
658	dp = data_read_id(dp, &id);
	Value stored to 'dp' is never read
659	return id;
660	}
661
662	const char *
663	repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
664	{
665	unsigned char *dp;
666	Repokey *key;
667	Id id;
668
669	dp = find_key_data(data, solvid, keyname, &key);
670	if (!dp)
671	return 0;
672	if (key->type == REPOKEY_TYPE_STR)
673	return (const char *)dp;
674	if (key->type == REPOKEY_TYPE_CONSTANTID)
675	id = key->size;
676	else if (key->type == REPOKEY_TYPE_ID)
677	dp = data_read_id(dp, &id);
678	else
679	return 0;
680	if (data->localpool)
681	return stringpool_id2str(&data->spool, id);
682	return pool_id2str(data->repo->pool, id);
683	}
684
685	int
686	repodata_lookup_num(Repodata data, Id solvid, Id keyname, unsigned long long value)
687	{
688	unsigned char *dp;
689	Repokey *key;
690	unsigned int high, low;
691
692	*value = 0;
693	dp = find_key_data(data, solvid, keyname, &key);
694	if (!dp)
695	return 0;
696	switch (key->type)
697	{
698	case REPOKEY_TYPE_NUM:
699	data_read_num64(dp, &low, &high);
700	*value = (unsigned long long)high << 32 \| low;
701	return 1;
702	case REPOKEY_TYPE_U32:
703	data_read_u32(dp, &low);
704	*value = low;
705	return 1;
706	case REPOKEY_TYPE_CONSTANT:
707	*value = key->size;
708	return 1;
709	default:
710	return 0;
711	}
712	}
713
714	int
715	repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
716	{
717	Id schema;
718	Id *keyp;
719	unsigned char *dp;
720
721	if (!maybe_load_repodata(data, keyname))
722	return 0;
723	dp = solvid2data(data, solvid, &schema);
724	if (!dp)
725	return 0;
726	/* can't use find_key_data as we need to test the type */
727	for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
728	if (data->keys[keyp].name == keyname && data->keys[keyp].type == REPOKEY_TYPE_VOID)
729	return 1;
730	return 0;
731	}
732
733	const unsigned char *
734	repodata_lookup_bin_checksum(Repodata data, Id solvid, Id keyname, Id typep)
735	{
736	unsigned char *dp;
737	Repokey *key;
738
739	dp = find_key_data(data, solvid, keyname, &key);
740	if (!dp)
741	return 0;
742	switch (key->type)
743	{
744	case_CHKSUM_TYPEScase REPOKEY_TYPE_MD5: case REPOKEY_TYPE_SHA1: case REPOKEY_TYPE_SHA224 : case REPOKEY_TYPE_SHA256: case REPOKEY_TYPE_SHA384: case REPOKEY_TYPE_SHA512:
745	break;
746	default:
747	return 0;
748	}
749	*typep = key->type;
750	return dp;
751	}
752
753	int
754	repodata_lookup_idarray(Repodata data, Id solvid, Id keyname, Queue q)
755	{
756	unsigned char *dp;
757	Repokey *key;
758	Id id;
759	int eof = 0;
760
761	queue_empty(q);
762	dp = find_key_data(data, solvid, keyname, &key);
763	if (!dp)
764	return 0;
765	if (key->type != REPOKEY_TYPE_IDARRAY)
766	return 0;
767	for (;;)
768	{
769	dp = data_read_ideof(dp, &id, &eof);
770	queue_push(q, id);
771	if (eof)
772	break;
773	}
774	return 1;
775	}
776
777	const void *
778	repodata_lookup_binary(Repodata data, Id solvid, Id keyname, int lenp)
779	{
780	unsigned char *dp;
781	Repokey *key;
782	Id len;
783
784	dp = find_key_data(data, solvid, keyname, &key);
785	if (!dp \|\| key->type != REPOKEY_TYPE_BINARY)
786	{
787	*lenp = 0;
788	return 0;
789	}
790	dp = data_read_id(dp, &len);
791	*lenp = len;
792	return dp;
793	}
794
795	Id
796	repodata_globalize_id(Repodata *data, Id id, int create)
797	{
798	if (!id \|\| !data \|\| !data->localpool)
799	return id;
800	return pool_str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
801	}
802
803	Id
804	repodata_localize_id(Repodata *data, Id id, int create)
805	{
806	if (!id \|\| !data \|\| !data->localpool)
807	return id;
808	return stringpool_str2id(&data->spool, pool_id2str(data->repo->pool, id), create);
809	}
810
811	Id
812	repodata_translate_id(Repodata data, Repodata fromdata, Id id, int create)
813	{
814	if (!id \|\| !data \|\| !fromdata)
815	return id;
816	if (!data->localpool \|\| !fromdata->localpool)
817	{
818	if (fromdata->localpool)
819	id = repodata_globalize_id(fromdata, id, create);
820	if (data->localpool)
821	id = repodata_localize_id(data, id, create);
822	return id;
823	}
824	/* localpool is set in both data and fromdata */
825	return stringpool_str2id(&data->spool, stringpool_id2str(&fromdata->spool, id), create);
826	}
827
828	Id
829	repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id voidid)
830	{
831	Id *ap;
832	if (!data->attrs)
833	return 0;
834	ap = data->attrs[solvid - data->start];
835	if (!ap)
836	return 0;
837	for (; *ap; ap += 2)
838	{
839	if (data->keys[*ap].name != keyname)
840	continue;
841	if (data->keys[*ap].type == REPOKEY_TYPE_VOID)
842	return voidid;
843	if (data->keys[*ap].type == REPOKEY_TYPE_ID)
844	return ap[1];
845	return 0;
846	}
847	return 0;
848	}
849
850	const char *
851	repodata_lookup_dirstrarray_uninternalized(Repodata data, Id solvid, Id keyname, Id didp, Id *iterp)
852	{
853	Id *ap, did;
854	Id iter = *iterp;
855	if (iter == 0) /* find key data */
856	{
857	if (!data->attrs)
858	return 0;
859	ap = data->attrs[solvid - data->start];
860	if (!ap)
861	return 0;
862	for (; *ap; ap += 2)
863	if (data->keys[ap].name == keyname && data->keys[ap].type == REPOKEY_TYPE_DIRSTRARRAY)
864	break;
865	if (!*ap)
866	return 0;
867	iter = ap[1];
868	}
869	did = *didp;
870	for (ap = data->attriddata + iter; *ap; ap += 2)
871	{
872	if (did && ap[0] != did)
873	continue;
874	*didp = ap[0];
875	*iterp = ap - data->attriddata + 2;
876	return (const char *)data->attrdata + ap[1];
877	}
878	*iterp = 0;
879	return 0;
880	}
881
882	const unsigned char *
883	repodata_lookup_bin_checksum_uninternalized(Repodata data, Id solvid, Id keyname, Id typep)
884	{
885	Id *ap;
886	if (!data->attrs)
887	return 0;
888	ap = data->attrs[solvid - data->start];
889	if (!ap)
890	return 0;
891	for (; *ap; ap += 2)
892	{
893	if (data->keys[*ap].name != keyname)
894	continue;
895	switch (data->keys[*ap].type)
896	{
897	case_CHKSUM_TYPEScase REPOKEY_TYPE_MD5: case REPOKEY_TYPE_SHA1: case REPOKEY_TYPE_SHA224 : case REPOKEY_TYPE_SHA256: case REPOKEY_TYPE_SHA384: case REPOKEY_TYPE_SHA512:
898	typep = data->keys[ap].type;
899	return (const unsigned char *)data->attrdata + ap[1];
900	default:
901	break;
902	}
903	}
904	return 0;
905	}
906
907	/************************************************************************
908	* data search
909	*/
910
911
912	const char *
913	repodata_stringify(Pool pool, Repodata data, Repokey key, KeyValue kv, int flags)
914	{
915	switch (key->type)
916	{
917	case REPOKEY_TYPE_ID:
918	case REPOKEY_TYPE_CONSTANTID:
919	case REPOKEY_TYPE_IDARRAY:
920	if (data && data->localpool)
921	kv->str = stringpool_id2str(&data->spool, kv->id);
922	else
923	kv->str = pool_id2str(pool, kv->id);
924	if ((flags & SEARCH_SKIP_KIND(1<<16)) != 0 && key->storage == KEY_STORAGE_SOLVABLE1)
925	{
926	const char *s;
927	for (s = kv->str; s >= 'a' && s <= 'z'; s++)
928	;
929	if (*s == ':' && s > kv->str)
930	kv->str = s + 1;
931	}
932	return kv->str;
933	case REPOKEY_TYPE_STR:
934	return kv->str;
935	case REPOKEY_TYPE_DIRSTRARRAY:
936	if (!(flags & SEARCH_FILES(1<<17)))
937	return kv->str; /* match just the basename */
938	if (kv->num)
939	return kv->str; /* already stringified */
940	/* Put the full filename into kv->str. */
941	kv->str = repodata_dir2str(data, kv->id, kv->str);
942	kv->num = 1; /* mark stringification */
943	return kv->str;
944	case_CHKSUM_TYPEScase REPOKEY_TYPE_MD5: case REPOKEY_TYPE_SHA1: case REPOKEY_TYPE_SHA224 : case REPOKEY_TYPE_SHA256: case REPOKEY_TYPE_SHA384: case REPOKEY_TYPE_SHA512:
945	if (!(flags & SEARCH_CHECKSUMS(1<<18)))
946	return 0; /* skip em */
947	if (kv->num)
948	return kv->str; /* already stringified */
949	kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
950	kv->num = 1; /* mark stringification */
951	return kv->str;
952	default:
953	return 0;
954	}
955	}
956
957
958	struct subschema_data {
959	Solvable *s;
960	void *cbdata;
961	KeyValue *parent;
962	};
963
964	/* search a specific repodata */
965	void
966	repodata_search(Repodata data, Id solvid, Id keyname, int flags, int (callback)(void cbdata, Solvable s, Repodata data, Repokey key, KeyValue kv), void cbdata)
967	{
968	Id schema;
969	Repokey *key;
970	Id keyid, kp, keyp;
971	unsigned char dp, ddp;
972	int onekey = 0;
973	int stop;
974	KeyValue kv;
975	Solvable *s;
976
977	if (!maybe_load_repodata(data, keyname))
978	return;
979	if (solvid == SOLVID_SUBSCHEMA-3)
980	{
981	struct subschema_data *subd = cbdata;
982	cbdata = subd->cbdata;
983	s = subd->s;
984	schema = subd->parent->id;
985	dp = (unsigned char *)subd->parent->str;
986	kv.parent = subd->parent;
987	}
988	else
989	{
990	schema = 0;
991	dp = solvid2data(data, solvid, &schema);
992	if (!dp)
993	return;
994	s = data->repo->pool->solvables + solvid;
995	kv.parent = 0;
996	}
997	keyp = data->schemadata + data->schemata[schema];
998	if (keyname)
999	{
1000	/* search for a specific key */
1001	for (kp = keyp; *kp; kp++)
1002	if (data->keys[*kp].name == keyname)
1003	break;
1004	if (!*kp)
1005	return;
1006	dp = forward_to_key(data, *kp, keyp, dp);
1007	if (!dp)
1008	return;
1009	keyp = kp;
1010	onekey = 1;
1011	}
1012	while ((keyid = *keyp++) != 0)
1013	{
1014	stop = 0;
1015	key = data->keys + keyid;
1016	ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
1017
1018	if (key->type == REPOKEY_TYPE_DELETED)
1019	continue;
1020	if (key->type == REPOKEY_TYPE_FLEXARRAY \|\| key->type == REPOKEY_TYPE_FIXARRAY)
1021	{
1022	struct subschema_data subd;
1023	int nentries;
1024	Id schema = 0;
1025
1026	subd.cbdata = cbdata;
1027	subd.s = s;
1028	subd.parent = &kv;
1029	ddp = data_read_id(ddp, &nentries);
1030	kv.num = nentries;
1031	kv.entry = 0;
1032	kv.eof = 0;
1033	while (ddp && nentries > 0)
1034	{
1035	if (!--nentries)
1036	kv.eof = 1;
1037	if (key->type == REPOKEY_TYPE_FLEXARRAY \|\| !kv.entry)
1038	ddp = data_read_id(ddp, &schema);
1039	kv.id = schema;
1040	kv.str = (char *)ddp;
1041	stop = callback(cbdata, s, data, key, &kv);
1042	if (stop > SEARCH_NEXT_KEY1)
1043	return;
1044	if (stop && stop != SEARCH_ENTERSUB-1)
1045	break;
1046	if ((flags & SEARCH_SUB(1<<9)) != 0 \|\| stop == SEARCH_ENTERSUB-1)
1047	repodata_search(data, SOLVID_SUBSCHEMA-3, 0, flags, callback, &subd);
1048	ddp = data_skip_schema(data, ddp, schema);
1049	kv.entry++;
1050	}
1051	if (!nentries && (flags & SEARCH_ARRAYSENTINEL(1<<10)) != 0)
1052	{
1053	/* sentinel */
1054	kv.eof = 2;
1055	kv.str = (char *)ddp;
1056	stop = callback(cbdata, s, data, key, &kv);
1057	if (stop > SEARCH_NEXT_KEY1)
1058	return;
1059	}
1060	if (onekey)
1061	return;
1062	continue;
1063	}
1064	kv.entry = 0;
1065	do
1066	{
1067	ddp = data_fetch(ddp, &kv, key);
1068	if (!ddp)
1069	break;
1070	stop = callback(cbdata, s, data, key, &kv);
1071	kv.entry++;
1072	}
1073	while (!kv.eof && !stop);
1074	if (onekey \|\| stop > SEARCH_NEXT_KEY1)
1075	return;
1076	}
1077	}
1078
1079	void
1080	repodata_setpos_kv(Repodata data, KeyValue kv)
1081	{
1082	Pool *pool = data->repo->pool;
1083	if (!kv)
1084	pool_clear_pos(pool);
1085	else
1086	{
1087	pool->pos.repo = data->repo;
1088	pool->pos.repodataid = data - data->repo->repodata;
1089	pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
1090	pool->pos.schema = kv->id;
1091	}
1092	}
1093
1094	/************************************************************************
1095	* data iterator functions
1096	*/
1097
1098	static inline Id *
1099	solvabledata_fetch(Solvable s, KeyValue kv, Id keyname)
1100	{
1101	kv->id = keyname;
1102	switch (keyname)
1103	{
1104	case SOLVABLE_NAME:
1105	kv->eof = 1;
1106	return &s->name;
1107	case SOLVABLE_ARCH:
1108	kv->eof = 1;
1109	return &s->arch;
1110	case SOLVABLE_EVR:
1111	kv->eof = 1;
1112	return &s->evr;
1113	case SOLVABLE_VENDOR:
1114	kv->eof = 1;
1115	return &s->vendor;
1116	case SOLVABLE_PROVIDES:
1117	kv->eof = 0;
1118	return s->provides ? s->repo->idarraydata + s->provides : 0;
1119	case SOLVABLE_OBSOLETES:
1120	kv->eof = 0;
1121	return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
1122	case SOLVABLE_CONFLICTS:
1123	kv->eof = 0;
1124	return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
1125	case SOLVABLE_REQUIRES:
1126	kv->eof = 0;
1127	return s->requires ? s->repo->idarraydata + s->requires : 0;
1128	case SOLVABLE_RECOMMENDS:
1129	kv->eof = 0;
1130	return s->recommends ? s->repo->idarraydata + s->recommends : 0;
1131	case SOLVABLE_SUPPLEMENTS:
1132	kv->eof = 0;
1133	return s->supplements ? s->repo->idarraydata + s->supplements : 0;
1134	case SOLVABLE_SUGGESTS:
1135	kv->eof = 0;
1136	return s->suggests ? s->repo->idarraydata + s->suggests : 0;
1137	case SOLVABLE_ENHANCES:
1138	kv->eof = 0;
1139	return s->enhances ? s->repo->idarraydata + s->enhances : 0;
1140	case RPM_RPMDBID:
1141	kv->eof = 1;
1142	return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
1143	default:
1144	return 0;
1145	}
1146	}
1147
1148	int
1149	datamatcher_init(Datamatcher ma, const char match, int flags)
1150	{
1151	match = match ? solv_strdup(match) : 0;
1152	ma->match = match;
1153	ma->flags = flags;
1154	ma->error = 0;
1155	ma->matchdata = 0;
1156	if ((flags & SEARCH_STRINGMASK15) == SEARCH_REGEX6)
1157	{
1158	ma->matchdata = solv_calloc(1, sizeof(regex_t));
1159	ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED1 \| REG_NOSUB(((1 << 1) << 1) << 1) \| REG_NEWLINE((1 << 1) << 1) \| ((flags & SEARCH_NOCASE(1<<7)) ? REG_ICASE(1 << 1) : 0));
1160	if (ma->error)
1161	{
1162	solv_free(ma->matchdata);
1163	ma->flags = (flags & ~SEARCH_STRINGMASK15) \| SEARCH_ERROR15;
1164	}
1165	}
1166	if ((flags & SEARCH_FILES(1<<17)) != 0 && match)
1167	{
1168	/* prepare basename check */
1169	if ((flags & SEARCH_STRINGMASK15) == SEARCH_STRING1 \|\| (flags & SEARCH_STRINGMASK15) == SEARCH_STRINGEND3)
1170	{
1171	const char *p = strrchr(match, '/');
1172	ma->matchdata = (void *)(p ? p + 1 : match);
1173	}
1174	else if ((flags & SEARCH_STRINGMASK15) == SEARCH_GLOB5)
1175	{
1176	const char *p;
1177	for (p = match + strlen(match) - 1; p >= match; p--)
1178	if (p == '[' \|\| p == ']' \|\| p == '' \|\| p == '?' \|\| p == '/')
1179	break;
1180	ma->matchdata = (void *)(p + 1);
1181	}
1182	}
1183	return ma->error;
1184	}
1185
1186	void
1187	datamatcher_free(Datamatcher *ma)
1188	{
1189	if (ma->match)
1190	ma->match = solv_free((char *)ma->match);
1191	if ((ma->flags & SEARCH_STRINGMASK15) == SEARCH_REGEX6 && ma->matchdata)
1192	{
1193	regfree(ma->matchdata);
1194	solv_free(ma->matchdata);
1195	}
1196	ma->matchdata = 0;
1197	}
1198
1199	int
1200	datamatcher_match(Datamatcher ma, const char str)
1201	{
1202	int l;
1203	switch ((ma->flags & SEARCH_STRINGMASK15))
1204	{
1205	case SEARCH_SUBSTRING4:
1206	if (ma->flags & SEARCH_NOCASE(1<<7))
1207	return strcasestr(str, ma->match) != 0;
1208	else
1209	return strstr(str, ma->match) != 0;
1210	case SEARCH_STRING1:
1211	if (ma->flags & SEARCH_NOCASE(1<<7))
1212	return !strcasecmp(ma->match, str);
1213	else
1214	return !strcmp(ma->match, str);
1215	case SEARCH_STRINGSTART2:
1216	if (ma->flags & SEARCH_NOCASE(1<<7))
1217	return !strncasecmp(ma->match, str, strlen(ma->match));
1218	else
1219	return !strncmp(ma->match, str, strlen(ma->match));
1220	case SEARCH_STRINGEND3:
1221	l = strlen(str) - strlen(ma->match);
1222	if (l < 0)
1223	return 0;
1224	if (ma->flags & SEARCH_NOCASE(1<<7))
1225	return !strcasecmp(ma->match, str + l);
1226	else
1227	return !strcmp(ma->match, str + l);
1228	case SEARCH_GLOB5:
1229	return !fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE(1<<7)) ? FNM_CASEFOLD(1 << 4) : 0);
1230	case SEARCH_REGEX6:
1231	return !regexec((const regex_t )ma->matchdata, str, 0, NULL((void)0), 0);
1232	default:
1233	return 0;
1234	}
1235	}
1236
1237	/* check if the matcher can match the provides basename */
1238
1239	int
1240	datamatcher_checkbasename(Datamatcher ma, const char basename)
1241	{
1242	int l;
1243	const char *match = ma->matchdata;
1244	if (!match)
1245	return 1;
1246	switch (ma->flags & SEARCH_STRINGMASK15)
1247	{
1248	case SEARCH_STRING1:
1249	break;
1250	case SEARCH_STRINGEND3:
1251	if (match != ma->match)
1252	break; /* had slash, do exact match on basename */
1253	/* FALLTHROUGH */
1254	case SEARCH_GLOB5:
1255	/* check if the basename ends with match */
1256	l = strlen(basename) - strlen(match);
1257	if (l < 0)
1258	return 0;
1259	basename += l;
1260	break;
1261	default:
1262	return 1; /* maybe matches */
1263	}
1264	if ((ma->flags & SEARCH_NOCASE(1<<7)) != 0)
1265	return !strcasecmp(match, basename);
1266	else
1267	return !strcmp(match, basename);
1268	}
1269
1270	int
1271	repodata_filelistfilter_matches(Repodata data, const char str)
1272	{
1273	/* '.bin\/.', '^\/etc\/.', '^\/usr\/lib\/sendmail$' /
1274	/* for now hardcoded */
1275	if (strstr(str, "bin/"))
1276	return 1;
1277	if (!strncmp(str, "/etc/", 5))
1278	return 1;
1279	if (!strcmp(str, "/usr/lib/sendmail"))
1280	return 1;
1281	return 0;
1282	}
1283
1284
1285	enum {
1286	di_bye,
1287
1288	di_enterrepo,
1289	di_entersolvable,
1290	di_enterrepodata,
1291	di_enterschema,
1292	di_enterkey,
1293
1294	di_nextattr,
1295	di_nextkey,
1296	di_nextrepodata,
1297	di_nextsolvable,
1298	di_nextrepo,
1299
1300	di_enterarray,
1301	di_nextarrayelement,
1302
1303	di_entersub,
1304	di_leavesub,
1305
1306	di_nextsolvablekey,
1307	di_entersolvablekey,
1308	di_nextsolvableattr
1309	};
1310
1311	/* see dataiterator.h for documentation */
1312	int
1313	dataiterator_init(Dataiterator di, Pool pool, Repo repo, Id p, Id keyname, const char match, int flags)
1314	{
1315	memset(di, 0, sizeof(*di));
1316	di->pool = pool;
1317	di->flags = flags & ~SEARCH_THISSOLVID(1<<31);
1318	if (!pool \|\| (repo && repo->pool != pool))
1319	{
1320	di->state = di_bye;
1321	return -1;
1322	}
1323	if (match)
1324	{
1325	int error;
1326	if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1327	{
1328	di->state = di_bye;
1329	return error;
1330	}
1331	}
1332	di->keyname = keyname;
1333	di->keynames[0] = keyname;
1334	dataiterator_set_search(di, repo, p);
1335	return 0;
1336	}
1337
1338	void
1339	dataiterator_init_clone(Dataiterator di, Dataiterator from)
1340	{
1341	di = from;
1342	if (di->dupstr)
1343	{
1344	if (di->dupstr == di->kv.str)
1345	di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
1346	else
1347	{
1348	di->dupstr = 0;
1349	di->dupstrn = 0;
1350	}
1351	}
1352	memset(&di->matcher, 0, sizeof(di->matcher));
1353	if (from->matcher.match)
1354	datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
1355	if (di->nparents)
1356	{
1357	/* fix pointers */
1358	int i;
1359	for (i = 1; i < di->nparents; i++)
1360	di->parents[i].kv.parent = &di->parents[i - 1].kv;
1361	di->kv.parent = &di->parents[di->nparents - 1].kv;
1362	}
1363	}
1364
1365	int
1366	dataiterator_set_match(Dataiterator di, const char match, int flags)
1367	{
1368	di->flags = (flags & ~SEARCH_THISSOLVID(1<<31)) \| (di->flags & SEARCH_THISSOLVID(1<<31));
1369	datamatcher_free(&di->matcher);
1370	memset(&di->matcher, 0, sizeof(di->matcher));
1371	if (match)
1372	{
1373	int error;
1374	if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1375	{
1376	di->state = di_bye;
1377	return error;
1378	}
1379	}
1380	return 0;
1381	}
1382
1383	void
1384	dataiterator_set_search(Dataiterator di, Repo repo, Id p)
1385	{
1386	di->repo = repo;
1387	di->repoid = 0;
1388	di->flags &= ~SEARCH_THISSOLVID(1<<31);
1389	di->nparents = 0;
1390	di->rootlevel = 0;
1391	di->repodataid = 1;
1392	if (!di->pool->urepos)
1393	{
1394	di->state = di_bye;
1395	return;
1396	}
1397	if (!repo)
1398	{
1399	di->repoid = 1;
1400	di->repo = di->pool->repos[di->repoid];
1401	}
1402	di->state = di_enterrepo;
1403	if (p)
1404	dataiterator_jump_to_solvid(di, p);
1405	}
1406
1407	void
1408	dataiterator_set_keyname(Dataiterator *di, Id keyname)
1409	{
1410	di->nkeynames = 0;
1411	di->keyname = keyname;
1412	di->keynames[0] = keyname;
1413	}
1414
1415	void
1416	dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
1417	{
1418	int i;
1419
1420	if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
1421	{
1422	di->state = di_bye; /* sorry */
1423	return;
1424	}
1425	for (i = di->nkeynames + 1; i > 0; i--)
1426	di->keynames[i] = di->keynames[i - 1];
1427	di->keynames[0] = di->keyname = keyname;
1428	di->nkeynames++;
1429	}
1430
1431	void
1432	dataiterator_free(Dataiterator *di)
1433	{
1434	if (di->matcher.match)
1435	datamatcher_free(&di->matcher);
1436	if (di->dupstr)
1437	solv_free(di->dupstr);
1438	}
1439
1440	static unsigned char *
1441	dataiterator_find_keyname(Dataiterator *di, Id keyname)
1442	{
1443	Id *keyp;
1444	Repokey keys = di->data->keys, key;
1445	unsigned char *dp;
1446
1447	for (keyp = di->keyp; *keyp; keyp++)
1448	if (keys[*keyp].name == keyname)
1449	break;
1450	if (!*keyp)
1451	return 0;
1452	key = keys + *keyp;
1453	if (key->type == REPOKEY_TYPE_DELETED)
1454	return 0;
1455	if (key->storage != KEY_STORAGE_INCORE2 && key->storage != KEY_STORAGE_VERTICAL_OFFSET3)
1456	return 0; /* get_data will not work, no need to forward */
1457	dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1458	if (!dp)
1459	return 0;
1460	di->keyp = keyp;
1461	return dp;
1462	}
1463
1464	static inline int
1465	is_filelist_extension(Repodata *data)
1466	{
1467	int j;
1468	if (!repodata_precheck_keyname(data, SOLVABLE_FILELIST))
1469	return 0;
1470	for (j = 1; j < data->nkeys; j++)
1471	if (data->keys[j].name == SOLVABLE_FILELIST)
1472	break;
1473	if (j == data->nkeys)
1474	return 0;
1475	if (data->state != REPODATA_AVAILABLE0)
1476	return 1;
1477	for (j = 1; j < data->nkeys; j++)
1478	if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1479	return 0;
1480	return 1;
1481	}
1482
1483	static int
1484	dataiterator_filelistcheck(Dataiterator *di)
1485	{
1486	int j;
1487	int needcomplete = 0;
1488	Repodata *data = di->data;
1489
1490	if ((di->flags & SEARCH_COMPLETE_FILELIST(1<<12)) != 0)
1491	if (!di->matcher.match
1492	\|\| ((di->matcher.flags & (SEARCH_STRINGMASK15\|SEARCH_NOCASE(1<<7))) != SEARCH_STRING1
1493	&& (di->matcher.flags & (SEARCH_STRINGMASK15\|SEARCH_NOCASE(1<<7))) != SEARCH_GLOB5)
1494	\|\| !repodata_filelistfilter_matches(data, di->matcher.match))
1495	needcomplete = 1;
1496	if (data->state != REPODATA_AVAILABLE0)
1497	return needcomplete ? 1 : 0;
1498	if (!needcomplete)
1499	{
1500	/* we don't need the complete filelist, so ignore all stubs */
1501	if (data->repo->nrepodata == 2)
1502	return 1;
1503	for (j = 1; j < data->nkeys; j++)
1504	if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1505	return 1;
1506	return 0;
1507	}
1508	else
1509	{
1510	/* we need the complete filelist. check if we habe a filtered filelist and there's
1511	* a extension with the complete filelist later on */
1512	for (j = 1; j < data->nkeys; j++)
1513	if (data->keys[j].name == SOLVABLE_FILELIST)
1514	break;
1515	if (j == data->nkeys)
1516	return 0; /* does not have filelist */
1517	for (j = 1; j < data->nkeys; j++)
1518	if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1519	break;
1520	if (j == data->nkeys)
1521	return 1; /* this is the externsion */
1522	while (data - data->repo->repodata + 1 < data->repo->nrepodata)
1523	{
1524	data++;
1525	if (is_filelist_extension(data))
1526	return 0;
1527	}
1528	return 1;
1529	}
1530	}
1531
1532	int
1533	dataiterator_step(Dataiterator *di)
1534	{
1535	Id schema;
1536
1537	if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET3 && di->vert_ddp && di->vert_storestate != di->data->storestate) {
1538	unsigned int ddpoff = di->ddp - di->vert_ddp;
1539	di->vert_off += ddpoff;
1540	di->vert_len -= ddpoff;
1541	di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len);
1542	di->vert_storestate = di->data->storestate;
1543	if (!di->ddp)
1544	di->state = di_nextkey;
1545	}
1546	for (;;)
1547	{
1548	switch (di->state)
1549	{
1550	case di_enterrepo: di_enterrepo:
1551	if (!di->repo \|\| (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS(1<<11))))
1552	goto di_nextrepo;
1553	if (!(di->flags & SEARCH_THISSOLVID(1<<31)))
1554	{
1555	di->solvid = di->repo->start - 1; /* reset solvid iterator */
1556	goto di_nextsolvable;
1557	}
1558	/* FALLTHROUGH */
1559
1560	case di_entersolvable: di_entersolvable:
1561	if (di->repodataid)
1562	{
1563	di->repodataid = 1; /* reset repodata iterator */
1564	if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE(1<<8)) && (!di->keyname \|\| (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
1565	{
1566	extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1];
1567
1568	di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1569	di->data = 0;
1570	goto di_entersolvablekey;
1571	}
1572	}
1573	/* FALLTHROUGH */
1574
1575	case di_enterrepodata: di_enterrepodata:
1576	if (di->repodataid)
1577	{
1578	if (di->repodataid >= di->repo->nrepodata)
1579	goto di_nextsolvable;
1580	di->data = di->repo->repodata + di->repodataid;
1581	}
1582	if (di->repodataid && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
1583	goto di_nextrepodata;
1584	if (!maybe_load_repodata(di->data, di->keyname))
1585	goto di_nextrepodata;
1586	di->dp = solvid2data(di->data, di->solvid, &schema);
1587	if (!di->dp)
1588	goto di_nextrepodata;
1589	if (di->solvid == SOLVID_POS-2)
1590	di->solvid = di->pool->pos.solvid;
1591	/* reset key iterator */
1592	di->keyp = di->data->schemadata + di->data->schemata[schema];
1593	/* FALLTHROUGH */
1594
1595	case di_enterschema: di_enterschema:
1596	if (di->keyname)
1597	di->dp = dataiterator_find_keyname(di, di->keyname);
1598	if (!di->dp \|\| !*di->keyp)
1599	{
1600	if (di->kv.parent)
1601	goto di_leavesub;
1602	goto di_nextrepodata;
1603	}
1604	/* FALLTHROUGH */
1605
1606	case di_enterkey: di_enterkey:
1607	di->kv.entry = -1;
1608	di->key = di->data->keys + *di->keyp;
1609	if (!di->dp)
1610	goto di_nextkey;
1611	/* this is get_data() modified to store vert_ data */
1612	if (di->key->storage == KEY_STORAGE_VERTICAL_OFFSET3)
1613	{
1614	Id off, len;
1615	di->dp = data_read_id(di->dp, &off);
1616	di->dp = data_read_id(di->dp, &len);
1617	di->vert_ddp = di->ddp = get_vertical_data(di->data, di->key, off, len);
1618	di->vert_off = off;
1619	di->vert_len = len;
1620	di->vert_storestate = di->data->storestate;
1621	}
1622	else if (di->key->storage == KEY_STORAGE_INCORE2)
1623	{
1624	di->ddp = di->dp;
1625	if (di->keyp[1] && (!di->keyname \|\| (di->flags & SEARCH_SUB(1<<9)) != 0))
1626	di->dp = data_skip_key(di->data, di->dp, di->key);
1627	}
1628	else
1629	di->ddp = 0;
1630	if (!di->ddp)
1631	goto di_nextkey;
1632	if (di->key->type == REPOKEY_TYPE_DELETED)
1633	goto di_nextkey;
1634	if (di->key->type == REPOKEY_TYPE_FIXARRAY \|\| di->key->type == REPOKEY_TYPE_FLEXARRAY)
1635	goto di_enterarray;
1636	if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1637	goto di_nextkey;
1638	/* FALLTHROUGH */
1639
1640	case di_nextattr:
1641	di->kv.entry++;
1642	di->ddp = data_fetch(di->ddp, &di->kv, di->key);
1643	if (di->kv.eof)
1644	di->state = di_nextkey;
1645	else
1646	di->state = di_nextattr;
1647	break;
1648
1649	case di_nextkey: di_nextkey:
1650	if (!di->keyname && *++di->keyp)
1651	goto di_enterkey;
1652	if (di->kv.parent)
1653	goto di_leavesub;
1654	/* FALLTHROUGH */
1655
1656	case di_nextrepodata: di_nextrepodata:
1657	if (di->repodataid && ++di->repodataid < di->repo->nrepodata)
1658	goto di_enterrepodata;
1659	/* FALLTHROUGH */
1660
1661	case di_nextsolvable: di_nextsolvable:
1662	if (!(di->flags & SEARCH_THISSOLVID(1<<31)))
1663	{
1664	if (di->solvid < 0)
1665	di->solvid = di->repo->start;
1666	else
1667	di->solvid++;
1668	for (; di->solvid < di->repo->end; di->solvid++)
1669	{
1670	if (di->pool->solvables[di->solvid].repo == di->repo)
1671	goto di_entersolvable;
1672	}
1673	}
1674	/* FALLTHROUGH */
1675
1676	case di_nextrepo: di_nextrepo:
1677	if (di->repoid > 0)
1678	{
1679	di->repoid++;
1680	di->repodataid = 1;
1681	if (di->repoid < di->pool->nrepos)
1682	{
1683	di->repo = di->pool->repos[di->repoid];
1684	goto di_enterrepo;
1685	}
1686	}
1687	/* FALLTHROUGH */
1688
1689	case di_bye: di_bye:
1690	di->state = di_bye;
1691	return 0;
1692
1693	case di_enterarray: di_enterarray:
1694	if (di->key->name == REPOSITORY_SOLVABLES)
1695	goto di_nextkey;
1696	di->ddp = data_read_id(di->ddp, (Id *)&di->kv.num);
1697	di->kv.eof = 0;
1698	di->kv.entry = -1;
1699	/* FALLTHROUGH */
1700
1701	case di_nextarrayelement: di_nextarrayelement:
1702	di->kv.entry++;
1703	if (di->kv.entry)
1704	di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1705	if (di->kv.entry == di->kv.num)
1706	{
1707	if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1708	goto di_nextkey;
1709	if (!(di->flags & SEARCH_ARRAYSENTINEL(1<<10)))
1710	goto di_nextkey;
1711	di->kv.str = (char *)di->ddp;
1712	di->kv.eof = 2;
1713	di->state = di_nextkey;
1714	break;
1715	}
1716	if (di->kv.entry == di->kv.num - 1)
1717	di->kv.eof = 1;
1718	if (di->key->type == REPOKEY_TYPE_FLEXARRAY \|\| !di->kv.entry)
1719	di->ddp = data_read_id(di->ddp, &di->kv.id);
1720	di->kv.str = (char *)di->ddp;
1721	if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1722	goto di_entersub;
1723	if ((di->flags & SEARCH_SUB(1<<9)) != 0)
1724	di->state = di_entersub;
1725	else
1726	di->state = di_nextarrayelement;
1727	break;
1728
1729	case di_entersub: di_entersub:
1730	if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1731	goto di_nextarrayelement; /* sorry, full */
1732	di->parents[di->nparents].kv = di->kv;
1733	di->parents[di->nparents].dp = di->dp;
1734	di->parents[di->nparents].keyp = di->keyp;
1735	di->dp = (unsigned char *)di->kv.str;
1736	di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1737	memset(&di->kv, 0, sizeof(di->kv));
1738	di->kv.parent = &di->parents[di->nparents].kv;
1739	di->nparents++;
1740	di->keyname = di->keynames[di->nparents - di->rootlevel];
1741	goto di_enterschema;
1742
1743	case di_leavesub: di_leavesub:
1744	if (di->nparents - 1 < di->rootlevel)
1745	goto di_bye;
1746	di->nparents--;
1747	di->dp = di->parents[di->nparents].dp;
1748	di->kv = di->parents[di->nparents].kv;
1749	di->keyp = di->parents[di->nparents].keyp;
1750	di->key = di->data->keys + *di->keyp;
1751	di->ddp = (unsigned char *)di->kv.str;
1752	di->keyname = di->keynames[di->nparents - di->rootlevel];
1753	goto di_nextarrayelement;
1754
1755	/* special solvable attr handling follows */
1756
1757	case di_nextsolvablekey: di_nextsolvablekey:
1758	if (di->keyname \|\| di->key->name == RPM_RPMDBID)
1759	goto di_enterrepodata;
1760	di->key++;
1761	/* FALLTHROUGH */
1762
1763	case di_entersolvablekey: di_entersolvablekey:
1764	di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
1765	if (!di->idp \|\| !*di->idp)
1766	goto di_nextsolvablekey;
1767	if (di->kv.eof)
1768	{
1769	/* not an array */
1770	di->kv.id = *di->idp;
1771	di->kv.num = di->idp; / for rpmdbid */
1772	di->kv.num2 = 0; /* for rpmdbid */
1773	di->kv.entry = 0;
1774	di->state = di_nextsolvablekey;
1775	break;
1776	}
1777	di->kv.entry = -1;
1778	/* FALLTHROUGH */
1779
1780	case di_nextsolvableattr:
1781	di->state = di_nextsolvableattr;
1782	di->kv.id = *di->idp++;
1783	di->kv.entry++;
1784	if (!*di->idp)
1785	{
1786	di->kv.eof = 1;
1787	di->state = di_nextsolvablekey;
1788	}
1789	break;
1790
1791	}
1792
1793	if (di->matcher.match)
1794	{
1795	const char *str;
1796	/* simple pre-check so that we don't need to stringify */
1797	if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES(1<<17)) != 0)
1798	if (!datamatcher_checkbasename(&di->matcher, di->kv.str))
1799	continue;
1800	if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
1801	{
1802	if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY \|\| di->key->type == REPOKEY_TYPE_FLEXARRAY))
1803	return 1;
1804	continue;
1805	}
1806	if (!datamatcher_match(&di->matcher, str))
1807	continue;
1808	}
1809	else
1810	{
1811	if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->flags & SEARCH_FILES(1<<17)) != 0)
1812	repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags);
1813	}
1814	/* found something! */
1815	return 1;
1816	}
1817	}
1818
1819	void
1820	dataiterator_entersub(Dataiterator *di)
1821	{
1822	if (di->state == di_nextarrayelement)
1823	di->state = di_entersub;
1824	}
1825
1826	void
1827	dataiterator_setpos(Dataiterator *di)
1828	{
1829	if (di->kv.eof == 2)
1830	{
1831	pool_clear_pos(di->pool);
1832	return;
1833	}
1834	di->pool->pos.solvid = di->solvid;
1835	di->pool->pos.repo = di->repo;
1836	di->pool->pos.repodataid = di->data - di->repo->repodata;
1837	di->pool->pos.schema = di->kv.id;
1838	di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1839	}
1840
1841	void
1842	dataiterator_setpos_parent(Dataiterator *di)
1843	{
1844	if (!di->kv.parent \|\| di->kv.parent->eof == 2)
1845	{
1846	pool_clear_pos(di->pool);
1847	return;
1848	}
1849	di->pool->pos.solvid = di->solvid;
1850	di->pool->pos.repo = di->repo;
1851	di->pool->pos.repodataid = di->data - di->repo->repodata;
1852	di->pool->pos.schema = di->kv.parent->id;
1853	di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
1854	}
1855
1856	/* clones just the position, not the search keys/matcher */
1857	void
1858	dataiterator_clonepos(Dataiterator di, Dataiterator from)
1859	{
1860	di->state = from->state;
1861	di->flags &= ~SEARCH_THISSOLVID(1<<31);
1862	di->flags \|= (from->flags & SEARCH_THISSOLVID(1<<31));
1863	di->repo = from->repo;
1864	di->data = from->data;
1865	di->dp = from->dp;
1866	di->ddp = from->ddp;
1867	di->idp = from->idp;
1868	di->keyp = from->keyp;
1869	di->key = from->key;
1870	di->kv = from->kv;
1871	di->repodataid = from->repodataid;
1872	di->solvid = from->solvid;
1873	di->repoid = from->repoid;
1874	di->rootlevel = from->rootlevel;
1875	memcpy(di->parents, from->parents, sizeof(from->parents));
1876	di->nparents = from->nparents;
1877	if (di->nparents)
1878	{
1879	int i;
1880	for (i = 1; i < di->nparents; i++)
1881	di->parents[i].kv.parent = &di->parents[i - 1].kv;
1882	di->kv.parent = &di->parents[di->nparents - 1].kv;
1883	}
1884	di->dupstr = 0;
1885	di->dupstrn = 0;
1886	if (from->dupstr && from->dupstr == from->kv.str)
1887	{
1888	di->dupstrn = from->dupstrn;
1889	di->dupstr = solv_memdup(from->dupstr, from->dupstrn);
1890	}
1891	}
1892
1893	void
1894	dataiterator_seek(Dataiterator *di, int whence)
1895	{
1896	if ((whence & DI_SEEK_STAY(1 << 16)) != 0)
1897	di->rootlevel = di->nparents;
1898	switch (whence & ~DI_SEEK_STAY(1 << 16))
1899	{
1900	case DI_SEEK_CHILD1:
1901	if (di->state != di_nextarrayelement)
1902	break;
1903	if ((whence & DI_SEEK_STAY(1 << 16)) != 0)
1904	di->rootlevel = di->nparents + 1; /* XXX: dangerous! */
1905	di->state = di_entersub;
1906	break;
1907	case DI_SEEK_PARENT2:
1908	if (!di->nparents)
1909	{
1910	di->state = di_bye;
1911	break;
1912	}
1913	di->nparents--;
1914	if (di->rootlevel > di->nparents)
1915	di->rootlevel = di->nparents;
1916	di->dp = di->parents[di->nparents].dp;
1917	di->kv = di->parents[di->nparents].kv;
1918	di->keyp = di->parents[di->nparents].keyp;
1919	di->key = di->data->keys + *di->keyp;
1920	di->ddp = (unsigned char *)di->kv.str;
1921	di->keyname = di->keynames[di->nparents - di->rootlevel];
1922	di->state = di_nextarrayelement;
1923	break;
1924	case DI_SEEK_REWIND3:
1925	if (!di->nparents)
1926	{
1927	di->state = di_bye;
1928	break;
1929	}
1930	di->dp = (unsigned char *)di->kv.parent->str;
1931	di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
1932	di->state = di_enterschema;
1933	break;
1934	default:
1935	break;
1936	}
1937	}
1938
1939	void
1940	dataiterator_skip_attribute(Dataiterator *di)
1941	{
1942	if (di->state == di_nextsolvableattr)
1943	di->state = di_nextsolvablekey;
1944	else
1945	di->state = di_nextkey;
1946	}
1947
1948	void
1949	dataiterator_skip_solvable(Dataiterator *di)
1950	{
1951	di->nparents = 0;
1952	di->kv.parent = 0;
1953	di->rootlevel = 0;
1954	di->keyname = di->keynames[0];
1955	di->state = di_nextsolvable;
1956	}
1957
1958	void
1959	dataiterator_skip_repo(Dataiterator *di)
1960	{
1961	di->nparents = 0;
1962	di->kv.parent = 0;
1963	di->rootlevel = 0;
1964	di->keyname = di->keynames[0];
1965	di->state = di_nextrepo;
1966	}
1967
1968	void
1969	dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
1970	{
1971	di->nparents = 0;
1972	di->kv.parent = 0;
1973	di->rootlevel = 0;
1974	di->keyname = di->keynames[0];
1975	if (solvid == SOLVID_POS-2)
1976	{
1977	di->repo = di->pool->pos.repo;
1978	if (!di->repo)
1979	{
1980	di->state = di_bye;
1981	return;
1982	}
1983	di->repoid = 0;
1984	if (!di->pool->pos.repodataid && di->pool->pos.solvid == SOLVID_META-1) {
1985	solvid = SOLVID_META-1; /* META pos hack */
1986	} else {
1987	di->data = di->repo->repodata + di->pool->pos.repodataid;
1988	di->repodataid = 0;
1989	}
1990	}
1991	else if (solvid > 0)
1992	{
1993	di->repo = di->pool->solvables[solvid].repo;
1994	di->repoid = 0;
1995	}
1996	if (di->repoid > 0)
1997	{
1998	if (!di->pool->urepos)
1999	{
2000	di->state = di_bye;
2001	return;
2002	}
2003	di->repoid = 1;
2004	di->repo = di->pool->repos[di->repoid];
2005	}
2006	if (solvid != SOLVID_POS-2)
2007	di->repodataid = 1;
2008	di->solvid = solvid;
2009	if (solvid)
2010	di->flags \|= SEARCH_THISSOLVID(1<<31);
2011	di->state = di_enterrepo;
2012	}
2013
2014	void
2015	dataiterator_jump_to_repo(Dataiterator di, Repo repo)
2016	{
2017	di->nparents = 0;
2018	di->kv.parent = 0;
2019	di->rootlevel = 0;
2020	di->repo = repo;
2021	di->repoid = 0; /* 0 means stay at repo */
2022	di->repodataid = 1;
2023	di->solvid = 0;
2024	di->flags &= ~SEARCH_THISSOLVID(1<<31);
2025	di->state = di_enterrepo;
2026	}
2027
2028	int
2029	dataiterator_match(Dataiterator di, Datamatcher ma)
2030	{
2031	const char *str;
2032	if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
2033	return 0;
2034	return ma ? datamatcher_match(ma, str) : 1;
2035	}
2036
2037	void
2038	dataiterator_strdup(Dataiterator *di)
2039	{
2040	int l = -1;
2041
2042	if (!di->kv.str \|\| di->kv.str == di->dupstr)
2043	return;
2044	switch (di->key->type)
2045	{
2046	case_CHKSUM_TYPEScase REPOKEY_TYPE_MD5: case REPOKEY_TYPE_SHA1: case REPOKEY_TYPE_SHA224 : case REPOKEY_TYPE_SHA256: case REPOKEY_TYPE_SHA384: case REPOKEY_TYPE_SHA512:
2047	case REPOKEY_TYPE_DIRSTRARRAY:
2048	if (di->kv.num) /* was it stringified into tmp space? */
2049	l = strlen(di->kv.str) + 1;
2050	break;
2051	default:
2052	break;
2053	}
2054	if (l < 0 && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET3)
2055	{
2056	switch (di->key->type)
2057	{
2058	case REPOKEY_TYPE_STR:
2059	case REPOKEY_TYPE_DIRSTRARRAY:
2060	l = strlen(di->kv.str) + 1;
2061	break;
2062	case_CHKSUM_TYPEScase REPOKEY_TYPE_MD5: case REPOKEY_TYPE_SHA1: case REPOKEY_TYPE_SHA224 : case REPOKEY_TYPE_SHA256: case REPOKEY_TYPE_SHA384: case REPOKEY_TYPE_SHA512:
2063	l = solv_chksum_len(di->key->type);
2064	break;
2065	case REPOKEY_TYPE_BINARY:
2066	l = di->kv.num;
2067	break;
2068	}
2069	}
2070	if (l >= 0)
2071	{
2072	if (!di->dupstrn \|\| di->dupstrn < l)
2073	{
2074	di->dupstrn = l + 16;
2075	di->dupstr = solv_realloc(di->dupstr, di->dupstrn);
2076	}
2077	if (l)
2078	memcpy(di->dupstr, di->kv.str, l);
2079	di->kv.str = di->dupstr;
2080	}
2081	}
2082
2083	/************************************************************************
2084	* data modify functions
2085	*/
2086
2087	/* extend repodata so that it includes solvables p */
2088	void
2089	repodata_extend(Repodata *data, Id p)
2090	{
2091	if (data->start == data->end)
2092	data->start = data->end = p;
2093	if (p >= data->end)
2094	{
2095	int old = data->end - data->start;
2096	int new = p - data->end + 1;
2097	if (data->attrs)
2098	{
2099	data->attrs = solv_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK255);
2100	memset(data->attrs + old, 0, new * sizeof(Id *));
2101	}
2102	data->incoreoffset = solv_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK255);
2103	memset(data->incoreoffset + old, 0, new * sizeof(Id));
2104	data->end = p + 1;
2105	}
2106	if (p < data->start)
2107	{
2108	int old = data->end - data->start;
2109	int new = data->start - p;
2110	if (data->attrs)
2111	{
2112	data->attrs = solv_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK255);
2113	memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
2114	memset(data->attrs, 0, new * sizeof(Id *));
2115	}
2116	data->incoreoffset = solv_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK255);
2117	memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
2118	memset(data->incoreoffset, 0, new * sizeof(Id));
2119	data->start = p;
2120	}
2121	}
2122
2123	/* shrink end of repodata */
2124	void
2125	repodata_shrink(Repodata *data, int end)
2126	{
2127	int i;
2128
2129	if (data->end <= end)
2130	return;
2131	if (data->start >= end)
2132	{
2133	if (data->attrs)
2134	{
2135	for (i = 0; i < data->end - data->start; i++)
2136	solv_free(data->attrs[i]);
2137	data->attrs = solv_free(data->attrs);
2138	}
2139	data->incoreoffset = solv_free(data->incoreoffset);
2140	data->start = data->end = 0;
2141	return;
2142	}
2143	if (data->attrs)
2144	{
2145	for (i = end; i < data->end; i++)
2146	solv_free(data->attrs[i - data->start]);
2147	data->attrs = solv_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK255);
2148	}
2149	if (data->incoreoffset)
2150	data->incoreoffset = solv_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK255);
2151	data->end = end;
2152	}
2153
2154	/* extend repodata so that it includes solvables from start to start + num - 1 */
2155	void
2156	repodata_extend_block(Repodata *data, Id start, Id num)
2157	{
2158	if (!num)
2159	return;
2160	if (!data->incoreoffset)
2161	{
2162	/* this also means that data->attrs is NULL */
2163	data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK255);
2164	data->start = start;
2165	data->end = start + num;
2166	return;
2167	}
2168	repodata_extend(data, start);
2169	if (num > 1)
2170	repodata_extend(data, start + num - 1);
2171	}
2172
2173	/**********************************************************************/
2174
2175
2176	#define REPODATA_ATTRS_BLOCK31 31
2177	#define REPODATA_ATTRDATA_BLOCK1023 1023
2178	#define REPODATA_ATTRIDDATA_BLOCK63 63
2179	#define REPODATA_ATTRNUM64DATA_BLOCK15 15
2180
2181
2182	Id
2183	repodata_new_handle(Repodata *data)
2184	{
2185	if (!data->nxattrs)
2186	{
2187	data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK255);
2188	data->nxattrs = 2; /* -1: SOLVID_META */
2189	}
2190	data->xattrs = solv_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK255);
2191	data->xattrs[data->nxattrs] = 0;
2192	return -(data->nxattrs++);
2193	}
2194
2195	static inline Id **
2196	repodata_get_attrp(Repodata *data, Id handle)
2197	{
2198	if (handle < 0)
2199	{
2200	if (handle == SOLVID_META-1 && !data->xattrs)
2201	{
2202	data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK255);
2203	data->nxattrs = 2;
2204	}
2205	return data->xattrs - handle;
2206	}
2207	if (handle < data->start \|\| handle >= data->end)
2208	repodata_extend(data, handle);
2209	if (!data->attrs)
2210	data->attrs = solv_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK255);
2211	return data->attrs + (handle - data->start);
2212	}
2213
2214	static void
2215	repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
2216	{
2217	Id *pp;
2218	Id ap, *app;
2219	int i;
2220
2221	app = repodata_get_attrp(data, handle);
2222	ap = *app;
2223	i = 0;
2224	if (ap)
2225	{
2226	/* Determine equality based on the name only, allows us to change
2227	type (when overwrite is set), and makes TYPE_CONSTANT work. */
2228	for (pp = ap; *pp; pp += 2)
2229	if (data->keys[*pp].name == data->keys[keyid].name)
2230	break;
2231	if (*pp)
2232	{
2233	if (overwrite \|\| data->keys[*pp].type == REPOKEY_TYPE_DELETED)
2234	{
2235	pp[0] = keyid;
2236	pp[1] = val;
2237	}
2238	return;
2239	}
2240	i = pp - ap;
2241	}
2242	ap = solv_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK31);
2243	*app = ap;
2244	pp = ap + i;
2245	*pp++ = keyid;
2246	*pp++ = val;
2247	*pp = 0;
2248	}
2249
2250
2251	static void
2252	repodata_set(Repodata data, Id solvid, Repokey key, Id val)
2253	{
2254	Id keyid;
2255
2256	keyid = repodata_key2id(data, key, 1);
2257	repodata_insert_keyid(data, solvid, keyid, val, 1);
2258	}
2259
2260	void
2261	repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
2262	{
2263	Repokey key;
2264	key.name = keyname;
2265	key.type = REPOKEY_TYPE_ID;
2266	key.size = 0;
2267	key.storage = KEY_STORAGE_INCORE2;
2268	repodata_set(data, solvid, &key, id);
2269	}
2270
2271	void
2272	repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned long long num)
2273	{
2274	Repokey key;
2275	key.name = keyname;
2276	key.type = REPOKEY_TYPE_NUM;
2277	key.size = 0;
2278	key.storage = KEY_STORAGE_INCORE2;
2279	if (num >= 0x80000000)
2280	{
2281	data->attrnum64data = solv_extend(data->attrnum64data, data->attrnum64datalen, 1, sizeof(unsigned long long), REPODATA_ATTRNUM64DATA_BLOCK15);
2282	data->attrnum64data[data->attrnum64datalen] = num;
2283	num = 0x80000000 \| data->attrnum64datalen++;
2284	}
2285	repodata_set(data, solvid, &key, (Id)num);
2286	}
2287
2288	void
2289	repodata_set_poolstr(Repodata data, Id solvid, Id keyname, const char str)
2290	{
2291	Repokey key;
2292	Id id;
2293	if (data->localpool)
2294	id = stringpool_str2id(&data->spool, str, 1);
2295	else
2296	id = pool_str2id(data->repo->pool, str, 1);
2297	key.name = keyname;
2298	key.type = REPOKEY_TYPE_ID;
2299	key.size = 0;
2300	key.storage = KEY_STORAGE_INCORE2;
2301	repodata_set(data, solvid, &key, id);
2302	}
2303
2304	void
2305	repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
2306	{
2307	Repokey key;
2308	key.name = keyname;
2309	key.type = REPOKEY_TYPE_CONSTANT;
2310	key.size = constant;
2311	key.storage = KEY_STORAGE_INCORE2;
2312	repodata_set(data, solvid, &key, 0);
2313	}
2314
2315	void
2316	repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
2317	{
2318	Repokey key;
2319	key.name = keyname;
2320	key.type = REPOKEY_TYPE_CONSTANTID;
2321	key.size = id;
2322	key.storage = KEY_STORAGE_INCORE2;
2323	repodata_set(data, solvid, &key, 0);
2324	}
2325
2326	void
2327	repodata_set_void(Repodata *data, Id solvid, Id keyname)
2328	{
2329	Repokey key;
2330	key.name = keyname;
2331	key.type = REPOKEY_TYPE_VOID;
2332	key.size = 0;
2333	key.storage = KEY_STORAGE_INCORE2;
2334	repodata_set(data, solvid, &key, 0);
2335	}
2336
2337	void
2338	repodata_set_str(Repodata data, Id solvid, Id keyname, const char str)
2339	{
2340	Repokey key;
2341	int l;
2342
2343	l = strlen(str) + 1;
2344	key.name = keyname;
2345	key.type = REPOKEY_TYPE_STR;
2346	key.size = 0;
2347	key.storage = KEY_STORAGE_INCORE2;
2348	data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK1023);
2349	memcpy(data->attrdata + data->attrdatalen, str, l);
2350	repodata_set(data, solvid, &key, data->attrdatalen);
2351	data->attrdatalen += l;
2352	}
2353
2354	void
2355	repodata_set_binary(Repodata data, Id solvid, Id keyname, void buf, int len)
2356	{
2357	Repokey key;
2358	unsigned char *dp;
2359
2360	if (len < 0)
2361	return;
2362	key.name = keyname;
2363	key.type = REPOKEY_TYPE_BINARY;
2364	key.size = 0;
2365	key.storage = KEY_STORAGE_INCORE2;
2366	data->attrdata = solv_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK1023);
2367	dp = data->attrdata + data->attrdatalen;
2368	if (len >= (1 << 14))
2369	{
2370	if (len >= (1 << 28))
2371	*dp++ = (len >> 28) \| 128;
2372	if (len >= (1 << 21))
2373	*dp++ = (len >> 21) \| 128;
2374	*dp++ = (len >> 14) \| 128;
2375	}
2376	if (len >= (1 << 7))
2377	*dp++ = (len >> 7) \| 128;
2378	*dp++ = len & 127;
2379	if (len)
2380	memcpy(dp, buf, len);
2381	repodata_set(data, solvid, &key, data->attrdatalen);
2382	data->attrdatalen = dp + len - data->attrdata;
2383	}
2384
2385	/* add an array element consisting of entrysize Ids to the repodata. modifies attriddata
2386	* so that the caller can append entrysize new elements plus the termination zero there */
2387	static void
2388	repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
2389	{
2390	int oldsize;
2391	Id ida, pp, **ppp;
2392
2393	/* check if it is the same as last time, this speeds things up a lot */
2394	if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
2395	{
2396	/* great! just append the new data */
2397	data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK63);
2398	data->attriddatalen--; /* overwrite terminating 0 */
2399	data->lastdatalen += entrysize;
2400	return;
2401	}
2402
2403	ppp = repodata_get_attrp(data, handle);
2404	pp = *ppp;
2405	if (pp)
2406	{
2407	for (; *pp; pp += 2)
2408	if (data->keys[*pp].name == keyname)
2409	break;
2410	}
2411	if (!pp \|\| !pp \|\| data->keys[pp].type != keytype)
2412	{
2413	/* not found. allocate new key */
2414	Repokey key;
2415	Id keyid;
2416	key.name = keyname;
2417	key.type = keytype;
2418	key.size = 0;
2419	key.storage = KEY_STORAGE_INCORE2;
2420	data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK63);
2421	keyid = repodata_key2id(data, &key, 1);
2422	repodata_insert_keyid(data, handle, keyid, data->attriddatalen, 1);
2423	data->lasthandle = handle;
2424	data->lastkey = keyid;
2425	data->lastdatalen = data->attriddatalen + entrysize + 1;
2426	return;
2427	}
2428	oldsize = 0;
2429	for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
2430	oldsize += entrysize;
2431	if (ida + 1 == data->attriddata + data->attriddatalen)
2432	{
2433	/* this was the last entry, just append it */
2434	data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK63);
2435	data->attriddatalen--; /* overwrite terminating 0 */
2436	}
2437	else
2438	{
2439	/* too bad. move to back. */
2440	data->attriddata = solv_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK63);
2441	memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
2442	pp[1] = data->attriddatalen;
2443	data->attriddatalen += oldsize;
2444	}
2445	data->lasthandle = handle;
2446	data->lastkey = *pp;
2447	data->lastdatalen = data->attriddatalen + entrysize + 1;
2448	}
2449
2450	void
2451	repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2452	const unsigned char *str)
2453	{
2454	Repokey key;
2455	int l;
2456
2457	if (!(l = solv_chksum_len(type)))
2458	return;
2459	key.name = keyname;
2460	key.type = type;
2461	key.size = 0;
2462	key.storage = KEY_STORAGE_INCORE2;
2463	data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK1023);
2464	memcpy(data->attrdata + data->attrdatalen, str, l);
2465	repodata_set(data, solvid, &key, data->attrdatalen);
2466	data->attrdatalen += l;
2467	}
2468
2469	void
2470	repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2471	const char *str)
2472	{
2473	unsigned char buf[64];
2474	int l;
2475
2476	if (!(l = solv_chksum_len(type)))
2477	return;
2478	if (l > sizeof(buf) \|\| solv_hex2bin(&str, buf, l) != l)
2479	return;
2480	repodata_set_bin_checksum(data, solvid, keyname, type, buf);
2481	}
2482
2483	const char *
2484	repodata_chk2str(Repodata data, Id type, const unsigned char buf)
2485	{
2486	int l;
2487
2488	if (!(l = solv_chksum_len(type)))
2489	return "";
2490	return pool_bin2hex(data->repo->pool, buf, l);
2491	}
2492
2493	/* rpm filenames don't contain the epoch, so strip it */
2494	static inline const char *
2495	evrid2vrstr(Pool *pool, Id evrid)
2496	{
2497	const char p, evr = pool_id2str(pool, evrid);
2498	if (!evr)
2499	return evr;
2500	for (p = evr; p >= '0' && p <= '9'; p++)
2501	;
2502	return p != evr && *p == ':' && p[1] ? p + 1 : evr;
2503	}
2504
2505	static inline void
2506	repodata_set_poolstrn(Repodata data, Id solvid, Id keyname, const char str, int l)
2507	{
2508	Id id;
2509	if (data->localpool)
2510	id = stringpool_strn2id(&data->spool, str, l, 1);
2511	else
2512	id = pool_strn2id(data->repo->pool, str, l, 1);
2513	repodata_set_id(data, solvid, keyname, id);
2514	}
2515
2516	static inline void
2517	repodata_set_strn(Repodata data, Id solvid, Id keyname, const char str, int l)
2518	{
2519	if (!str[l])
2520	repodata_set_str(data, solvid, keyname, str);
2521	else
2522	{
2523	char *s = solv_strdup(str);
2524	s[l] = 0;
2525	repodata_set_str(data, solvid, keyname, s);
2526	free(s);
2527	}
2528	}
2529
2530	void
2531	repodata_set_location(Repodata data, Id solvid, int medianr, const char dir, const char *file)
2532	{
2533	Pool *pool = data->repo->pool;
2534	Solvable *s;
2535	const char str, fp;
2536	int l = 0;
2537
2538	if (medianr)
2539	repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
2540	if (!dir)
2541	{
2542	if ((dir = strrchr(file, '/')) != 0)
2543	{
2544	l = dir - file;
2545	dir = file;
2546	file = dir + l + 1;
2547	if (!l)
2548	l++;
2549	}
2550	}
2551	else
2552	l = strlen(dir);
2553	if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 \|\| dir[2] != '/'))
2554	{
2555	dir += 2;
2556	l -= 2;
2557	}
2558	if (l == 1 && dir[0] == '.')
2559	l = 0;
2560	s = pool->solvables + solvid;
2561	if (dir && l)
2562	{
2563	str = pool_id2str(pool, s->arch);
2564	if (!strncmp(dir, str, l) && !str[l])
2565	repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
2566	else
2567	repodata_set_strn(data, solvid, SOLVABLE_MEDIADIR, dir, l);
2568	}
2569	fp = file;
2570	str = pool_id2str(pool, s->name);
2571	l = strlen(str);
2572	if ((!l \|\| !strncmp(fp, str, l)) && fp[l] == '-')
2573	{
2574	fp += l + 1;
2575	str = evrid2vrstr(pool, s->evr);
2576	l = strlen(str);
2577	if ((!l \|\| !strncmp(fp, str, l)) && fp[l] == '.')
2578	{
2579	fp += l + 1;
2580	str = pool_id2str(pool, s->arch);
2581	l = strlen(str);
2582	if ((!l \|\| !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
2583	{
2584	repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
2585	return;
2586	}
2587	}
2588	}
2589	repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
2590	}
2591
2592	/* XXX: medianr is currently not stored */
2593	void
2594	repodata_set_deltalocation(Repodata data, Id handle, int medianr, const char dir, const char *file)
2595	{
2596	int l = 0;
2597	const char evr, suf, *s;
2598
2599	if (!dir)
2600	{
2601	if ((dir = strrchr(file, '/')) != 0)
2602	{
2603	l = dir - file;
2604	dir = file;
2605	file = dir + l + 1;
2606	if (!l)
2607	l++;
2608	}
2609	}
2610	else
2611	l = strlen(dir);
2612	if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 \|\| dir[2] != '/'))
2613	{
2614	dir += 2;
2615	l -= 2;
2616	}
2617	if (l == 1 && dir[0] == '.')
2618	l = 0;
2619	if (dir && l)
2620	repodata_set_poolstrn(data, handle, DELTA_LOCATION_DIR, dir, l);
2621	evr = strchr(file, '-');
2622	if (evr)
2623	{
2624	for (s = evr - 1; s > file; s--)
2625	if (*s == '-')
2626	{
2627	evr = s;
2628	break;
2629	}
2630	}
2631	suf = strrchr(file, '.');
2632	if (suf)
2633	{
2634	for (s = suf - 1; s > file; s--)
2635	if (*s == '.')
2636	{
2637	suf = s;
2638	break;
2639	}
2640	if (!strcmp(suf, ".delta.rpm") \|\| !strcmp(suf, ".patch.rpm"))
2641	{
2642	/* We accept one more item as suffix. */
2643	for (s = suf - 1; s > file; s--)
2644	if (*s == '.')
2645	{
2646	suf = s;
2647	break;
2648	}
2649	}
2650	}
2651	if (!evr)
2652	suf = 0;
2653	if (suf && evr && suf < evr)
2654	suf = 0;
2655	repodata_set_poolstrn(data, handle, DELTA_LOCATION_NAME, file, evr ? evr - file : strlen(file));
2656	if (evr)
2657	repodata_set_poolstrn(data, handle, DELTA_LOCATION_EVR, evr + 1, suf ? suf - evr - 1: strlen(evr + 1));
2658	if (suf)
2659	repodata_set_poolstr(data, handle, DELTA_LOCATION_SUFFIX, suf + 1);
2660	}
2661
2662	void
2663	repodata_set_sourcepkg(Repodata data, Id solvid, const char sourcepkg)
2664	{
2665	Pool *pool = data->repo->pool;
2666	Solvable *s = pool->solvables + solvid;
2667	const char p, sevr, sarch, name, *evr;
2668
2669	p = strrchr(sourcepkg, '.');
2670	if (!p \|\| strcmp(p, ".rpm") != 0)
2671	{
2672	if (*sourcepkg)
2673	repodata_set_str(data, solvid, SOLVABLE_SOURCENAME, sourcepkg);
2674	return;
2675	}
2676	p--;
2677	while (p > sourcepkg && *p != '.')
2678	p--;
2679	if (*p != '.' \|\| p == sourcepkg)
2680	return;
2681	sarch = p-- + 1;
2682	while (p > sourcepkg && *p != '-')
2683	p--;
2684	if (*p != '-' \|\| p == sourcepkg)
2685	return;
2686	p--;
2687	while (p > sourcepkg && *p != '-')
2688	p--;
2689	if (*p != '-' \|\| p == sourcepkg)
2690	return;
2691	sevr = p + 1;
2692	pool = s->repo->pool;
2693
2694	name = pool_id2str(pool, s->name);
2695	if (name && !strncmp(sourcepkg, name, sevr - sourcepkg - 1) && name[sevr - sourcepkg - 1] == 0)
2696	repodata_set_void(data, solvid, SOLVABLE_SOURCENAME);
2697	else
2698	repodata_set_id(data, solvid, SOLVABLE_SOURCENAME, pool_strn2id(pool, sourcepkg, sevr - sourcepkg - 1, 1));
2699
2700	evr = evrid2vrstr(pool, s->evr);
2701	if (evr && !strncmp(sevr, evr, sarch - sevr - 1) && evr[sarch - sevr - 1] == 0)
2702	repodata_set_void(data, solvid, SOLVABLE_SOURCEEVR);
2703	else
2704	repodata_set_id(data, solvid, SOLVABLE_SOURCEEVR, pool_strn2id(pool, sevr, sarch - sevr - 1, 1));
2705
2706	if (!strcmp(sarch, "src.rpm"))
2707	repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_SRC);
2708	else if (!strcmp(sarch, "nosrc.rpm"))
2709	repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_NOSRC);
2710	else
2711	repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, pool_strn2id(pool, sarch, strlen(sarch) - 4, 1));
2712	}
2713
2714	void
2715	repodata_set_idarray(Repodata data, Id solvid, Id keyname, Queue q)
2716	{
2717	Repokey key;
2718	int i;
2719
2720	key.name = keyname;
2721	key.type = REPOKEY_TYPE_IDARRAY;
2722	key.size = 0;
2723	key.storage = KEY_STORAGE_INCORE2;
2724	repodata_set(data, solvid, &key, data->attriddatalen);
2725	data->attriddata = solv_extend(data->attriddata, data->attriddatalen, q->count + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK63);
2726	for (i = 0; i < q->count; i++)
2727	data->attriddata[data->attriddatalen++] = q->elements[i];
2728	data->attriddata[data->attriddatalen++] = 0;
2729	}
2730
2731	void
2732	repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
2733	{
2734	assert(dir)({ if (dir) ; else __assert_fail ("dir", "/home/brain/Projects/upstream/libsolv/src/repodata.c" , 2734, __PRETTY_FUNCTION__); });
2735	#if 0
2736	fprintf(stderrstderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
2737	#endif
2738	repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2739	data->attriddata[data->attriddatalen++] = dir;
2740	data->attriddata[data->attriddatalen++] = num;
2741	data->attriddata[data->attriddatalen++] = num2;
2742	data->attriddata[data->attriddatalen++] = 0;
2743	}
2744
2745	void
2746	repodata_add_dirstr(Repodata data, Id solvid, Id keyname, Id dir, const char str)
2747	{
2748	Id stroff;
2749	int l;
2750
2751	assert(dir)({ if (dir) ; else __assert_fail ("dir", "/home/brain/Projects/upstream/libsolv/src/repodata.c" , 2751, __PRETTY_FUNCTION__); });
2752	l = strlen(str) + 1;
2753	data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK1023);
2754	memcpy(data->attrdata + data->attrdatalen, str, l);
2755	stroff = data->attrdatalen;
2756	data->attrdatalen += l;
2757
2758	#if 0
2759	fprintf(stderrstderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str, data->attriddatalen);
2760	#endif
2761	repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2762	data->attriddata[data->attriddatalen++] = dir;
2763	data->attriddata[data->attriddatalen++] = stroff;
2764	data->attriddata[data->attriddatalen++] = 0;
2765	}
2766
2767	void
2768	repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
2769	{
2770	#if 0
2771	fprintf(stderrstderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
2772	#endif
2773	repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
2774	data->attriddata[data->attriddatalen++] = id;
2775	data->attriddata[data->attriddatalen++] = 0;
2776	}
2777
2778	void
2779	repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
2780	const char *str)
2781	{
2782	Id id;
2783	if (data->localpool)
2784	id = stringpool_str2id(&data->spool, str, 1);
2785	else
2786	id = pool_str2id(data->repo->pool, str, 1);
2787	repodata_add_idarray(data, solvid, keyname, id);
2788	}
2789
2790	void
2791	repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2792	{
2793	repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2794	data->attriddata[data->attriddatalen++] = ghandle;
2795	data->attriddata[data->attriddatalen++] = 0;
2796	}
2797
2798	void
2799	repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2800	{
2801	repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2802	data->attriddata[data->attriddatalen++] = ghandle;
2803	data->attriddata[data->attriddatalen++] = 0;
2804	}
2805
2806	void
2807	repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname)
2808	{
2809	Id pp, ap, **app;
2810	app = repodata_get_attrp(data, solvid);
2811	ap = *app;
2812	if (!ap)
2813	return;
2814	for (; *ap; ap += 2)
2815	if (data->keys[*ap].name == keyname)
2816	break;
2817	if (!*ap)
2818	return;
2819	pp = ap;
2820	ap += 2;
2821	for (; *ap; ap += 2)
2822	{
2823	if (data->keys[*ap].name == keyname)
2824	continue;
2825	*pp++ = ap[0];
2826	*pp++ = ap[1];
2827	}
2828	*pp = 0;
2829	}
2830
2831	/* XXX: does not work correctly, needs fix in iterators! */
2832	void
2833	repodata_unset(Repodata *data, Id solvid, Id keyname)
2834	{
2835	Repokey key;
2836	key.name = keyname;
2837	key.type = REPOKEY_TYPE_DELETED;
2838	key.size = 0;
2839	key.storage = KEY_STORAGE_INCORE2;
2840	repodata_set(data, solvid, &key, 0);
2841	}
2842
2843	/* add all (uninternalized) attrs from src to dest */
2844	void
2845	repodata_merge_attrs(Repodata *data, Id dest, Id src)
2846	{
2847	Id *keyp;
2848	if (dest == src \|\| !data->attrs \|\| !(keyp = data->attrs[src - data->start]))
2849	return;
2850	for (; *keyp; keyp += 2)
2851	repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2852	}
2853
2854	/* add some (uninternalized) attrs from src to dest */
2855	void
2856	repodata_merge_some_attrs(Repodata data, Id dest, Id src, Map keyidmap, int overwrite)
2857	{
2858	Id *keyp;
2859	if (dest == src \|\| !data->attrs \|\| !(keyp = data->attrs[src - data->start]))
2860	return;
2861	for (; *keyp; keyp += 2)
2862	if (!keyidmap \|\| MAPTST(keyidmap, keyp[0])((keyidmap)->map[(keyp[0]) >> 3] & (1 << ( (keyp[0]) & 7))))
2863	repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
2864	}
2865
2866	/* swap (uninternalized) attrs from src and dest */
2867	void
2868	repodata_swap_attrs(Repodata *data, Id dest, Id src)
2869	{
2870	Id *tmpattrs;
2871	if (!data->attrs \|\| dest == src)
2872	return;
2873	if (dest < data->start \|\| dest >= data->end)
2874	repodata_extend(data, dest);
2875	if (src < data->start \|\| src >= data->end)
2876	repodata_extend(data, src);
2877	tmpattrs = data->attrs[dest - data->start];
2878	data->attrs[dest - data->start] = data->attrs[src - data->start];
2879	data->attrs[src - data->start] = tmpattrs;
2880	}
2881
2882
2883	/**********************************************************************/
2884
2885	/* TODO: unify with repo_write and repo_solv! */
2886
2887	#define EXTDATA_BLOCK1023 1023
2888
2889	struct extdata {
2890	unsigned char *buf;
2891	int len;
2892	};
2893
2894	static void
2895	data_addid(struct extdata *xd, Id sx)
2896	{
2897	unsigned int x = (unsigned int)sx;
2898	unsigned char *dp;
2899
2900	xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK1023);
2901	dp = xd->buf + xd->len;
2902
2903	if (x >= (1 << 14))
2904	{
2905	if (x >= (1 << 28))
2906	*dp++ = (x >> 28) \| 128;
2907	if (x >= (1 << 21))
2908	*dp++ = (x >> 21) \| 128;
2909	*dp++ = (x >> 14) \| 128;
2910	}
2911	if (x >= (1 << 7))
2912	*dp++ = (x >> 7) \| 128;
2913	*dp++ = x & 127;
2914	xd->len = dp - xd->buf;
2915	}
2916
2917	static void
2918	data_addid64(struct extdata *xd, unsigned long long x)
2919	{
2920	if (x >= 0x100000000)
2921	{
2922	if ((x >> 35) != 0)
2923	{
2924	data_addid(xd, (Id)(x >> 35));
2925	xd->buf[xd->len - 1] \|= 128;
2926	}
2927	data_addid(xd, (Id)((unsigned int)x \| 0x80000000));
2928	xd->buf[xd->len - 5] = (x >> 28) \| 128;
2929	}
2930	else
2931	data_addid(xd, (Id)x);
2932	}
2933
2934	static void
2935	data_addideof(struct extdata *xd, Id sx, int eof)
2936	{
2937	unsigned int x = (unsigned int)sx;
2938	unsigned char *dp;
2939
2940	xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK1023);
2941	dp = xd->buf + xd->len;
2942
2943	if (x >= (1 << 13))
2944	{
2945	if (x >= (1 << 27))
2946	*dp++ = (x >> 27) \| 128;
2947	if (x >= (1 << 20))
2948	*dp++ = (x >> 20) \| 128;
2949	*dp++ = (x >> 13) \| 128;
2950	}
2951	if (x >= (1 << 6))
2952	*dp++ = (x >> 6) \| 128;
2953	*dp++ = eof ? (x & 63) : (x & 63) \| 64;
2954	xd->len = dp - xd->buf;
2955	}
2956
2957	static void
2958	data_addblob(struct extdata xd, unsigned char blob, int len)
2959	{
2960	xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK1023);
2961	memcpy(xd->buf + xd->len, blob, len);
2962	xd->len += len;
2963	}
2964
2965	/*********************************/
2966
2967	/* this is to reduct memory usage when internalizing oversized repos */
2968	static void
2969	compact_attrdata(Repodata *data, int entry, int nentry)
2970	{
2971	int i;
2972	unsigned int attrdatastart = data->attrdatalen;
2973	unsigned int attriddatastart = data->attriddatalen;
2974	if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
2975	return;
2976	for (i = entry; i < nentry; i++)
2977	{
2978	Id v, *attrs = data->attrs[i];
2979	if (!attrs)
2980	continue;
2981	for (; *attrs; attrs += 2)
2982	{
2983	switch (data->keys[*attrs].type)
2984	{
2985	case REPOKEY_TYPE_STR:
2986	case REPOKEY_TYPE_BINARY:
2987	case_CHKSUM_TYPEScase REPOKEY_TYPE_MD5: case REPOKEY_TYPE_SHA1: case REPOKEY_TYPE_SHA224 : case REPOKEY_TYPE_SHA256: case REPOKEY_TYPE_SHA384: case REPOKEY_TYPE_SHA512:
2988	if ((unsigned int)attrs[1] < attrdatastart)
2989	attrdatastart = attrs[1];
2990	break;
2991	case REPOKEY_TYPE_DIRSTRARRAY:
2992	for (v = attrs[1]; data->attriddata[v] ; v += 2)
2993	if (data->attriddata[v + 1] < attrdatastart)
2994	attrdatastart = data->attriddata[v + 1];
2995	/* FALLTHROUGH */
2996	case REPOKEY_TYPE_IDARRAY:
2997	case REPOKEY_TYPE_DIRNUMNUMARRAY:
2998	if ((unsigned int)attrs[1] < attriddatastart)
2999	attriddatastart = attrs[1];
3000	break;
3001	case REPOKEY_TYPE_FIXARRAY:
3002	case REPOKEY_TYPE_FLEXARRAY:
3003	return;
3004	default:
3005	break;
3006	}
3007	}
3008	}
3009	#if 0
3010	printf("compact_attrdata %d %d\n", entry, nentry);
3011	printf("attrdatastart: %d\n", attrdatastart);
3012	printf("attriddatastart: %d\n", attriddatastart);
3013	#endif
3014	if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
3015	return;
3016	for (i = entry; i < nentry; i++)
3017	{
3018	Id v, *attrs = data->attrs[i];
3019	if (!attrs)
3020	continue;
3021	for (; *attrs; attrs += 2)
3022	{
3023	switch (data->keys[*attrs].type)
3024	{
3025	case REPOKEY_TYPE_STR:
3026	case REPOKEY_TYPE_BINARY:
3027	case_CHKSUM_TYPEScase REPOKEY_TYPE_MD5: case REPOKEY_TYPE_SHA1: case REPOKEY_TYPE_SHA224 : case REPOKEY_TYPE_SHA256: case REPOKEY_TYPE_SHA384: case REPOKEY_TYPE_SHA512:
3028	attrs[1] -= attrdatastart;
3029	break;
3030	case REPOKEY_TYPE_DIRSTRARRAY:
3031	for (v = attrs[1]; data->attriddata[v] ; v += 2)
3032	data->attriddata[v + 1] -= attrdatastart;
3033	/* FALLTHROUGH */
3034	case REPOKEY_TYPE_IDARRAY:
3035	case REPOKEY_TYPE_DIRNUMNUMARRAY:
3036	attrs[1] -= attriddatastart;
3037	break;
3038	default:
3039	break;
3040	}
3041	}
3042	}
3043	if (attrdatastart)
3044	{
3045	data->attrdatalen -= attrdatastart;
3046	memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
3047	data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK1023);
3048	}
3049	if (attriddatastart)
3050	{
3051	data->attriddatalen -= attriddatastart;
3052	memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
3053	data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK63);
3054	}
3055	}
3056
3057	/* internalalize some key into incore/vincore data */
3058
3059	static void
3060	repodata_serialize_key(Repodata data, struct extdata newincore,
3061	struct extdata *newvincore,
3062	Id *schema,
3063	Repokey *key, Id val)
3064	{
3065	Id *ida;
3066	struct extdata *xd;
3067	unsigned int oldvincorelen = 0;
3068	Id schemaid, *sp;
3069
3070	xd = newincore;
3071	if (key->storage == KEY_STORAGE_VERTICAL_OFFSET3)
3072	{
3073	xd = newvincore;
3074	oldvincorelen = xd->len;
3075	}
3076	switch (key->type)
3077	{
3078	case REPOKEY_TYPE_VOID:
3079	case REPOKEY_TYPE_CONSTANT:
3080	case REPOKEY_TYPE_CONSTANTID:
3081	case REPOKEY_TYPE_DELETED:
3082	break;
3083	case REPOKEY_TYPE_STR:
3084	data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
3085	break;
3086	case REPOKEY_TYPE_MD5:
3087	data_addblob(xd, data->attrdata + val, SIZEOF_MD516);
3088	break;
3089	case REPOKEY_TYPE_SHA1:
3090	data_addblob(xd, data->attrdata + val, SIZEOF_SHA120);
3091	break;
3092	case REPOKEY_TYPE_SHA224:
3093	data_addblob(xd, data->attrdata + val, SIZEOF_SHA22428);
3094	break;
3095	case REPOKEY_TYPE_SHA256:
3096	data_addblob(xd, data->attrdata + val, SIZEOF_SHA25632);
3097	break;
3098	case REPOKEY_TYPE_SHA384:
3099	data_addblob(xd, data->attrdata + val, SIZEOF_SHA38448);
3100	break;
3101	case REPOKEY_TYPE_SHA512:
3102	data_addblob(xd, data->attrdata + val, SIZEOF_SHA51264);
3103	break;
3104	case REPOKEY_TYPE_NUM:
3105	if (val & 0x80000000)
3106	{
3107	data_addid64(xd, data->attrnum64data[val ^ 0x80000000]);
3108	break;
3109	}
3110	/* FALLTHROUGH */
3111	case REPOKEY_TYPE_ID:
3112	case REPOKEY_TYPE_DIR:
3113	data_addid(xd, val);
3114	break;
3115	case REPOKEY_TYPE_BINARY:
3116	{
3117	Id len;
3118	unsigned char *dp = data_read_id(data->attrdata + val, &len);
3119	dp += (unsigned int)len;
3120	data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val));
3121	}
3122	break;
3123	case REPOKEY_TYPE_IDARRAY:
3124	for (ida = data->attriddata + val; *ida; ida++)
3125	data_addideof(xd, ida[0], ida[1] ? 0 : 1);
3126	break;
3127	case REPOKEY_TYPE_DIRNUMNUMARRAY:
3128	for (ida = data->attriddata + val; *ida; ida += 3)
3129	{
3130	data_addid(xd, ida[0]);
3131	data_addid(xd, ida[1]);
3132	data_addideof(xd, ida[2], ida[3] ? 0 : 1);
3133	}
3134	break;
3135	case REPOKEY_TYPE_DIRSTRARRAY:
3136	for (ida = data->attriddata + val; *ida; ida += 2)
3137	{
3138	data_addideof(xd, ida[0], ida[2] ? 0 : 1);
3139	data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
3140	}
3141	break;
3142	case REPOKEY_TYPE_FIXARRAY:
3143	{
3144	int num = 0;
3145	schemaid = 0;
3146	for (ida = data->attriddata + val; *ida; ida++)
3147	{
3148	Id *kp;
3149	sp = schema;
3150	kp = data->xattrs[-*ida];
3151	if (!kp)
3152	continue; /* ignore empty elements */
3153	num++;
3154	for (; *kp; kp += 2)
3155	sp++ = kp;
3156	*sp = 0;
3157	if (!schemaid)
3158	schemaid = repodata_schema2id(data, schema, 1);
3159	else if (schemaid != repodata_schema2id(data, schema, 0))
3160	{
3161	pool_debug(data->repo->pool, SOLV_ERROR(1<<1), "repodata_serialize_key: fixarray substructs with different schemas\n");
3162	num = 0;
3163	break;
3164	}
3165	}
3166	data_addid(xd, num);
3167	if (!num)
3168	break;
3169	data_addid(xd, schemaid);
3170	for (ida = data->attriddata + val; *ida; ida++)
3171	{
3172	Id kp = data->xattrs[-ida];
3173	if (!kp)
3174	continue;
3175	for (; *kp; kp += 2)
3176	repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3177	}
3178	break;
3179	}
3180	case REPOKEY_TYPE_FLEXARRAY:
3181	{
3182	int num = 0;
3183	for (ida = data->attriddata + val; *ida; ida++)
3184	num++;
3185	data_addid(xd, num);
3186	for (ida = data->attriddata + val; *ida; ida++)
3187	{
3188	Id kp = data->xattrs[-ida];
3189	if (!kp)
3190	{
3191	data_addid(xd, 0); /* XXX */
3192	continue;
3193	}
3194	sp = schema;
3195	for (;*kp; kp += 2)
3196	sp++ = kp;
3197	*sp = 0;
3198	schemaid = repodata_schema2id(data, schema, 1);
3199	data_addid(xd, schemaid);
3200	kp = data->xattrs[-*ida];
3201	for (;*kp; kp += 2)
3202	repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3203	}
3204	break;
3205	}
3206	default:
3207	pool_debug(data->repo->pool, SOLV_FATAL(1<<0), "repodata_serialize_key: don't know how to handle type %d\n", key->type);
3208	exit(1);
3209	}
3210	if (key->storage == KEY_STORAGE_VERTICAL_OFFSET3)
3211	{
3212	/* put offset/len in incore */
3213	data_addid(newincore, data->lastverticaloffset + oldvincorelen);
3214	oldvincorelen = xd->len - oldvincorelen;
3215	data_addid(newincore, oldvincorelen);
3216	}
3217	}
3218
3219	/* create a circular linked list of all keys that share
3220	* the same keyname */
3221	static Id *
3222	calculate_keylink(Repodata *data)
3223	{
3224	int i, j;
3225	Id *link;
3226	Id maxkeyname = 0, *keytable = 0;
3227	link = solv_calloc(data->nkeys, sizeof(Id));
3228	if (data->nkeys <= 2)
3229	return link;
3230	for (i = 1; i < data->nkeys; i++)
3231	{
3232	Id n = data->keys[i].name;
3233	if (n >= maxkeyname)
3234	{
3235	keytable = solv_realloc2(keytable, n + 128, sizeof(Id));
3236	memset(keytable + maxkeyname, 0, (n + 128 - maxkeyname) * sizeof(Id));
3237	maxkeyname = n + 128;
3238	}
3239	j = keytable[n];
3240	if (j)
3241	link[i] = link[j];
3242	else
3243	j = i;
3244	link[j] = i;
3245	keytable[n] = i;
3246	}
3247	/* remove links that just point to themselfs */
3248	for (i = 1; i < data->nkeys; i++)
3249	if (link[i] == i)
3250	link[i] = 0;
3251	solv_free(keytable);
3252	return link;
3253	}
3254
3255	void
3256	repodata_internalize(Repodata *data)
3257	{
3258	Repokey *key, solvkey;
3259	Id entry, nentry;
3260	Id schemaid, keyid, schema, sp, oldschemaid, keyp, seen;
3261	Offset *oldincoreoffs = 0;
3262	int schemaidx;
3263	unsigned char dp, ndp;
3264	int neednewschema;
3265	struct extdata newincore;
3266	struct extdata newvincore;
3267	Id solvkeyid;
3268	Id *keylink;
3269	int haveoldkl;
3270
3271	if (!data->attrs && !data->xattrs)
3272	return;
3273
3274	#if 0
3275	printf("repodata_internalize %d\n", data->repodataid);
3276	printf(" attr data: %d K\n", data->attrdatalen / 1024);
3277	printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3278	#endif
3279	newvincore.buf = data->vincore;
3280	newvincore.len = data->vincorelen;
3281
3282	/* find the solvables key, create if needed */
3283	memset(&solvkey, 0, sizeof(solvkey));
3284	solvkey.name = REPOSITORY_SOLVABLES;
3285	solvkey.type = REPOKEY_TYPE_FLEXARRAY;
3286	solvkey.size = 0;
3287	solvkey.storage = KEY_STORAGE_INCORE2;
3288	solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
3289
3290	schema = solv_malloc2(data->nkeys, sizeof(Id));
3291	seen = solv_malloc2(data->nkeys, sizeof(Id));
3292
3293	/* Merge the data already existing (in data->schemata, ->incoredata and
3294	friends) with the new attributes in data->attrs[]. */
3295	nentry = data->end - data->start;
3296	memset(&newincore, 0, sizeof(newincore));
3297	data_addid(&newincore, 0); /* start data at offset 1 */
3298
3299	data->mainschema = 0;
3300	data->mainschemaoffsets = solv_free(data->mainschemaoffsets);
3301
3302	keylink = calculate_keylink(data);
3303	/* join entry data */
3304	/* we start with the meta data, entry -1 */
3305	for (entry = -1; entry < nentry; entry++)
3306	{
3307	oldschemaid = 0;
3308	dp = data->incoredata;
3309	if (dp)
3310	{
3311	dp += entry >= 0 ? data->incoreoffset[entry] : 1;
3312	dp = data_read_id(dp, &oldschemaid);
3313	}
3314	memset(seen, 0, data->nkeys * sizeof(Id));
3315	#if 0
3316	fprintf(stderrstderr, "oldschemaid %d\n", oldschemaid);
3317	fprintf(stderrstderr, "schemata %d\n", data->schemata[oldschemaid]);
3318	fprintf(stderrstderr, "schemadata %p\n", data->schemadata);
3319	#endif
3320
3321	/* seen: -1: old data, 0: skipped, >0: id + 1 */
3322	neednewschema = 0;
3323	sp = schema;
3324	haveoldkl = 0;
3325	for (keyp = data->schemadata + data->schemata[oldschemaid]; *keyp; keyp++)
3326	{
3327	if (seen[*keyp])
3328	{
3329	/* oops, should not happen */
3330	neednewschema = 1;
3331	continue;
3332	}
3333	seen[keyp] = -1; / use old marker */
3334	sp++ = keyp;
3335	if (keylink[*keyp])
3336	haveoldkl = 1; /* potential keylink conflict */
3337	}
3338
3339	/* strip solvables key */
3340	if (entry < 0 && solvkeyid && seen[solvkeyid])
3341	{
3342	*sp = 0;
3343	for (sp = keyp = schema; *sp; sp++)
3344	if (*sp != solvkeyid)
3345	keyp++ = sp;
3346	sp = keyp;
3347	seen[solvkeyid] = 0;
3348	neednewschema = 1;
3349	}
3350
3351	/* add new entries */
3352	if (entry >= 0)
3353	keyp = data->attrs ? data->attrs[entry] : 0;
3354	else
3355	keyp = data->xattrs ? data->xattrs[1] : 0;
3356	if (keyp)
3357	for (; *keyp; keyp += 2)
3358	{
3359	if (!seen[*keyp])
3360	{
3361	neednewschema = 1;
3362	sp++ = keyp;
3363	if (haveoldkl && keylink[keyp]) / this should be pretty rare */
3364	{
3365	Id kl;
3366	for (kl = keylink[keyp]; kl != keyp; kl = keylink[kl])
3367	if (seen[kl] == -1)
3368	{
3369	/* replacing old key kl, remove from schema and seen */
3370	Id *osp;
3371	for (osp = schema; osp < sp; osp++)
3372	if (*osp == kl)
3373	{
3374	memmove(osp, osp + 1, (sp - osp) * sizeof(Id));
3375	sp--;
3376	seen[kl] = 0;
3377	break;
3378	}
3379	}
3380	}
3381	}
3382	seen[*keyp] = keyp[1] + 1;
3383	}
3384
3385	/* add solvables key if needed */
3386	if (entry < 0 && data->end != data->start)
3387	{
3388	sp++ = solvkeyid; / always last in schema */
3389	neednewschema = 1;
3390	}
3391
3392	/* commit schema */
3393	*sp = 0;
3394	if (neednewschema)
3395	/* Ideally we'd like to sort the new schema here, to ensure
3396	schema equality independend of the ordering. */
3397	schemaid = repodata_schema2id(data, schema, 1);
3398	else
3399	schemaid = oldschemaid;
3400
3401	if (entry < 0)
3402	{
3403	data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
3404	data->mainschema = schemaid;
3405	}
3406
3407	/* find offsets in old incore data */
3408	if (oldschemaid)
3409	{
3410	Id *lastneeded = 0;
3411	for (sp = data->schemadata + data->schemata[oldschemaid]; *sp; sp++)
3412	if (seen[*sp] == -1)
3413	lastneeded = sp + 1;
3414	if (lastneeded)
3415	{
3416	if (!oldincoreoffs)
3417	oldincoreoffs = solv_malloc2(data->nkeys, 2 * sizeof(Offset));
3418	for (sp = data->schemadata + data->schemata[oldschemaid]; sp != lastneeded; sp++)
3419	{
3420	/* Skip the data associated with this old key. */
3421	key = data->keys + *sp;
3422	ndp = dp;
3423	if (key->storage == KEY_STORAGE_VERTICAL_OFFSET3)
3424	{
3425	ndp = data_skip(ndp, REPOKEY_TYPE_ID);
3426	ndp = data_skip(ndp, REPOKEY_TYPE_ID);
3427	}
3428	else if (key->storage == KEY_STORAGE_INCORE2)
3429	ndp = data_skip_key(data, ndp, key);
3430	oldincoreoffs[sp 2] = dp - data->incoredata;
3431	oldincoreoffs[sp 2 + 1] = ndp - dp;
3432	dp = ndp;
3433	}
3434	}
3435	}
3436
3437	/* just copy over the complete old entry (including the schemaid) if there was no new data */
3438	if (entry >= 0 && !neednewschema && oldschemaid && (!data->attrs \|\| !data->attrs[entry]) && dp)
3439	{
3440	ndp = data->incoredata + data->incoreoffset[entry];
3441	data->incoreoffset[entry] = newincore.len;
3442	data_addblob(&newincore, ndp, dp - ndp);
3443	goto entrydone;
3444	}
3445
3446	/* Now create data blob. We walk through the (possibly new) schema
3447	and either copy over old data, or insert the new. */
3448	if (entry >= 0)
3449	data->incoreoffset[entry] = newincore.len;
3450	data_addid(&newincore, schemaid);
3451
3452	/* we don't use a pointer to the schemadata here as repodata_serialize_key
3453	* may call repodata_schema2id() which might realloc our schemadata */
3454	for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
3455	{
3456	if (entry < 0)
3457	{
3458	data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
3459	if (keyid == solvkeyid)
3460	{
3461	/* add flexarray entry count */
3462	data_addid(&newincore, data->end - data->start);
3463	break; /* always the last entry */
3464	}
3465	}
3466	if (seen[keyid] == -1)
3467	{
3468	if (oldincoreoffs[keyid * 2 + 1])
3469	data_addblob(&newincore, data->incoredata + oldincoreoffs[keyid * 2], oldincoreoffs[keyid * 2 + 1]);
3470	}
3471	else if (seen[keyid])
3472	repodata_serialize_key(data, &newincore, &newvincore, schema, data->keys + keyid, seen[keyid] - 1);
3473	}
3474
3475	entrydone:
3476	/* free memory */
3477	if (entry >= 0 && data->attrs)
3478	{
3479	if (data->attrs[entry])
3480	data->attrs[entry] = solv_free(data->attrs[entry]);
3481	if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
3482	{
3483	compact_attrdata(data, entry + 1, nentry); /* try to free some memory */
3484	#if 0
3485	printf(" attr data: %d K\n", data->attrdatalen / 1024);
3486	printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3487	printf(" incore data: %d K\n", newincore.len / 1024);
3488	printf(" sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
3489	/* malloc_stats(); */
3490	#endif
3491	}
3492	}
3493	}
3494	/* free all xattrs */
3495	for (entry = 0; entry < data->nxattrs; entry++)
3496	if (data->xattrs[entry])
3497	solv_free(data->xattrs[entry]);
3498	data->xattrs = solv_free(data->xattrs);
3499	data->nxattrs = 0;
3500
3501	data->lasthandle = 0;
3502	data->lastkey = 0;
3503	data->lastdatalen = 0;
3504	solv_free(schema);
3505	solv_free(seen);
3506	solv_free(keylink);
3507	solv_free(oldincoreoffs);
3508	repodata_free_schemahash(data);
3509
3510	solv_free(data->incoredata);
3511	data->incoredata = newincore.buf;
3512	data->incoredatalen = newincore.len;
3513	data->incoredatafree = 0;
3514
3515	solv_free(data->vincore);
3516	data->vincore = newvincore.buf;
3517	data->vincorelen = newvincore.len;
3518
3519	data->attrs = solv_free(data->attrs);
3520	data->attrdata = solv_free(data->attrdata);
3521	data->attriddata = solv_free(data->attriddata);
3522	data->attrnum64data = solv_free(data->attrnum64data);
3523	data->attrdatalen = 0;
3524	data->attriddatalen = 0;
3525	data->attrnum64datalen = 0;
3526	#if 0
3527	printf("repodata_internalize %d done\n", data->repodataid);
3528	printf(" incore data: %d K\n", data->incoredatalen / 1024);
3529	#endif
3530	}
3531
3532	void
3533	repodata_disable_paging(Repodata *data)
3534	{
3535	if (maybe_load_repodata(data, 0))
3536	{
3537	repopagestore_disable_paging(&data->store);
3538	data->storestate++;
3539	}
3540	}
3541
3542	static void
3543	repodata_load_stub(Repodata *data)
3544	{
3545	Repo *repo = data->repo;
3546	Pool *pool = repo->pool;
3547	int r, i;
3548	struct _Pool_tmpspace oldtmpspace;
3549	Datapos oldpos;
3550
3551	if (!pool->loadcallback)
3552	{
3553	data->state = REPODATA_ERROR2;
3554	return;
3555	}
3556	data->state = REPODATA_LOADING4;
3557
3558	/* save tmp space and pos */
3559	oldtmpspace = pool->tmpspace;
3560	memset(&pool->tmpspace, 0, sizeof(pool->tmpspace));
3561	oldpos = pool->pos;
3562
3563	r = pool->loadcallback(pool, data, pool->loadcallbackdata);
3564
3565	/* restore tmp space and pos */
3566	for (i = 0; i < POOL_TMPSPACEBUF16; i++)
3567	solv_free(pool->tmpspace.buf[i]);
3568	pool->tmpspace = oldtmpspace;
3569	if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid)
3570	memset(&oldpos, 0, sizeof(oldpos));
3571	pool->pos = oldpos;
3572
3573	data->state = r ? REPODATA_AVAILABLE0 : REPODATA_ERROR2;
3574	}
3575
3576	static inline void
3577	repodata_add_stubkey(Repodata *data, Id keyname, Id keytype)
3578	{
3579	Repokey xkey;
3580
3581	xkey.name = keyname;
3582	xkey.type = keytype;
3583	xkey.storage = KEY_STORAGE_INCORE2;
3584	xkey.size = 0;
3585	repodata_key2id(data, &xkey, 1);
3586	}
3587
3588	static Repodata *
3589	repodata_add_stub(Repodata **datap)
3590	{
3591	Repodata data = datap;
3592	Repo *repo = data->repo;
3593	Id repodataid = data - repo->repodata;
3594	Repodata *sdata = repo_add_repodata(repo, 0);
3595	data = repo->repodata + repodataid;
3596	if (data->end > data->start)
3597	repodata_extend_block(sdata, data->start, data->end - data->start);
3598	sdata->state = REPODATA_STUB1;
3599	sdata->loadcallback = repodata_load_stub;
3600	*datap = data;
3601	return sdata;
3602	}
3603
3604	Repodata *
3605	repodata_create_stubs(Repodata *data)
3606	{
3607	Repo *repo = data->repo;
3608	Pool *pool = repo->pool;
3609	Repodata *sdata;
3610	int *stubdataids;
3611	Dataiterator di;
3612	Id xkeyname = 0;
3613	int i, cnt = 0;
3614
3615	dataiterator_init(&di, pool, repo, SOLVID_META-1, REPOSITORY_EXTERNAL, 0, 0);
3616	while (dataiterator_step(&di))
3617	if (di.data == data)
3618	cnt++;
3619	dataiterator_free(&di);
3620	if (!cnt)
3621	return data;
3622	stubdataids = solv_calloc(cnt, sizeof(*stubdataids));
3623	for (i = 0; i < cnt; i++)
3624	{
3625	sdata = repodata_add_stub(&data);
3626	stubdataids[i] = sdata - repo->repodata;
3627	}
3628	i = 0;
3629	dataiterator_init(&di, pool, repo, SOLVID_META-1, REPOSITORY_EXTERNAL, 0, 0);
3630	sdata = 0;
3631	while (dataiterator_step(&di))
3632	{
3633	if (di.data != data)
3634	continue;
3635	if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
3636	{
3637	dataiterator_entersub(&di);
3638	sdata = repo->repodata + stubdataids[i++];
3639	xkeyname = 0;
3640	continue;
3641	}
3642	switch (di.key->type)
3643	{
3644	case REPOKEY_TYPE_ID:
3645	repodata_set_id(sdata, SOLVID_META-1, di.key->name, di.kv.id);
3646	break;
3647	case REPOKEY_TYPE_CONSTANTID:
3648	repodata_set_constantid(sdata, SOLVID_META-1, di.key->name, di.kv.id);
3649	break;
3650	case REPOKEY_TYPE_STR:
3651	repodata_set_str(sdata, SOLVID_META-1, di.key->name, di.kv.str);
3652	break;
3653	case REPOKEY_TYPE_VOID:
3654	repodata_set_void(sdata, SOLVID_META-1, di.key->name);
3655	break;
3656	case REPOKEY_TYPE_NUM:
3657	repodata_set_num(sdata, SOLVID_META-1, di.key->name, SOLV_KV_NUM64(&di.kv)(((unsigned long long)((&di.kv)->num2)) << 32 \| ( &di.kv)->num));
3658	break;
3659	case_CHKSUM_TYPEScase REPOKEY_TYPE_MD5: case REPOKEY_TYPE_SHA1: case REPOKEY_TYPE_SHA224 : case REPOKEY_TYPE_SHA256: case REPOKEY_TYPE_SHA384: case REPOKEY_TYPE_SHA512:
3660	repodata_set_bin_checksum(sdata, SOLVID_META-1, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
3661	break;
3662	case REPOKEY_TYPE_IDARRAY:
3663	repodata_add_idarray(sdata, SOLVID_META-1, di.key->name, di.kv.id);
3664	if (di.key->name == REPOSITORY_KEYS)
3665	{
3666	if (!xkeyname)
3667	{
3668	if (!di.kv.eof)
3669	xkeyname = di.kv.id;
3670	}
3671	else
3672	{
3673	repodata_add_stubkey(sdata, xkeyname, di.kv.id);
3674	xkeyname = 0;
3675	}
3676	}
3677	break;
3678	default:
3679	break;
3680	}
3681	}
3682	dataiterator_free(&di);
3683	for (i = 0; i < cnt; i++)
3684	repodata_internalize(repo->repodata + stubdataids[i]);
3685	solv_free(stubdataids);
3686	return data;
3687	}
3688
3689	unsigned int
3690	repodata_memused(Repodata *data)
3691	{
3692	return data->incoredatalen + data->vincorelen;
3693	}
3694