Header And Logo
|
#include "postgres.h"#include "access/hash.h"#include "catalog/pg_operator.h"#include "commands/vacuum.h"#include "tsearch/ts_type.h"#include "utils/builtins.h"
Go to the source code of this file.
Data Structures | |
| struct | LexemeHashKey |
| struct | TrackItem |
Functions | |
| static void | compute_tsvector_stats (VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc, int samplerows, double totalrows) |
| static void | prune_lexemes_hashtable (HTAB *lexemes_tab, int b_current) |
| static uint32 | lexeme_hash (const void *key, Size keysize) |
| static int | lexeme_match (const void *key1, const void *key2, Size keysize) |
| static int | lexeme_compare (const void *key1, const void *key2) |
| static int | trackitem_compare_frequencies_desc (const void *e1, const void *e2) |
| static int | trackitem_compare_lexemes (const void *e1, const void *e2) |
| Datum | ts_typanalyze (PG_FUNCTION_ARGS) |
| static void compute_tsvector_stats | ( | VacAttrStats * | stats, | |
| AnalyzeAttrFetchFunc | fetchfunc, | |||
| int | samplerows, | |||
| double | totalrows | |||
| ) | [static] |
Definition at line 139 of file ts_typanalyze.c.
References VacAttrStats::anl_context, ARRPTR, Assert, VacAttrStats::attr, cstring_to_text_with_len(), CurrentMemoryContext, DatumGetPointer, DatumGetTSVector, DEBUG3, TrackItem::delta, elog, HASHCTL::entrysize, TrackItem::frequency, HASHCTL::hash, HASH_COMPARE, HASH_CONTEXT, hash_create(), HASH_ELEM, HASH_FUNCTION, hash_get_num_entries(), hash_search(), hash_seq_init(), hash_seq_search(), HASHCTL::hcxt, i, TrackItem::key, HASHCTL::keysize, WordEntry::len, LexemeHashKey::length, LexemeHashKey::lexeme, HASHCTL::match, Max, MemoryContextSwitchTo(), MemSet, Min, NULL, VacAttrStats::numnumbers, VacAttrStats::numvalues, palloc(), PointerGetDatum, WordEntry::pos, prune_lexemes_hashtable(), qsort, TSVectorData::size, VacAttrStats::stadistinct, VacAttrStats::stakind, VacAttrStats::stanullfrac, VacAttrStats::stanumbers, VacAttrStats::staop, VacAttrStats::stats_valid, VacAttrStats::statypalign, VacAttrStats::statypbyval, VacAttrStats::statypid, VacAttrStats::statyplen, VacAttrStats::stavalues, VacAttrStats::stawidth, STRPTR, trackitem_compare_frequencies_desc(), trackitem_compare_lexemes(), vacuum_delay_point(), value, and VARSIZE_ANY.
{
int num_mcelem;
int null_cnt = 0;
double total_width = 0;
/* This is D from the LC algorithm. */
HTAB *lexemes_tab;
HASHCTL hash_ctl;
HASH_SEQ_STATUS scan_status;
/* This is the current bucket number from the LC algorithm */
int b_current;
/* This is 'w' from the LC algorithm */
int bucket_width;
int vector_no,
lexeme_no;
LexemeHashKey hash_key;
TrackItem *item;
/*
* We want statistics_target * 10 lexemes in the MCELEM array. This
* multiplier is pretty arbitrary, but is meant to reflect the fact that
* the number of individual lexeme values tracked in pg_statistic ought to
* be more than the number of values for a simple scalar column.
*/
num_mcelem = stats->attr->attstattarget * 10;
/*
* We set bucket width equal to (num_mcelem + 10) / 0.007 as per the
* comment above.
*/
bucket_width = (num_mcelem + 10) * 1000 / 7;
/*
* Create the hashtable. It will be in local memory, so we don't need to
* worry about overflowing the initial size. Also we don't need to pay any
* attention to locking and memory management.
*/
MemSet(&hash_ctl, 0, sizeof(hash_ctl));
hash_ctl.keysize = sizeof(LexemeHashKey);
hash_ctl.entrysize = sizeof(TrackItem);
hash_ctl.hash = lexeme_hash;
hash_ctl.match = lexeme_match;
hash_ctl.hcxt = CurrentMemoryContext;
lexemes_tab = hash_create("Analyzed lexemes table",
num_mcelem,
&hash_ctl,
HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT);
/* Initialize counters. */
b_current = 1;
lexeme_no = 0;
/* Loop over the tsvectors. */
for (vector_no = 0; vector_no < samplerows; vector_no++)
{
Datum value;
bool isnull;
TSVector vector;
WordEntry *curentryptr;
char *lexemesptr;
int j;
vacuum_delay_point();
value = fetchfunc(stats, vector_no, &isnull);
/*
* Check for null/nonnull.
*/
if (isnull)
{
null_cnt++;
continue;
}
/*
* Add up widths for average-width calculation. Since it's a
* tsvector, we know it's varlena. As in the regular
* compute_minimal_stats function, we use the toasted width for this
* calculation.
*/
total_width += VARSIZE_ANY(DatumGetPointer(value));
/*
* Now detoast the tsvector if needed.
*/
vector = DatumGetTSVector(value);
/*
* We loop through the lexemes in the tsvector and add them to our
* tracking hashtable. Note: the hashtable entries will point into
* the (detoasted) tsvector value, therefore we cannot free that
* storage until we're done.
*/
lexemesptr = STRPTR(vector);
curentryptr = ARRPTR(vector);
for (j = 0; j < vector->size; j++)
{
bool found;
/* Construct a hash key */
hash_key.lexeme = lexemesptr + curentryptr->pos;
hash_key.length = curentryptr->len;
/* Lookup current lexeme in hashtable, adding it if new */
item = (TrackItem *) hash_search(lexemes_tab,
(const void *) &hash_key,
HASH_ENTER, &found);
if (found)
{
/* The lexeme is already on the tracking list */
item->frequency++;
}
else
{
/* Initialize new tracking list element */
item->frequency = 1;
item->delta = b_current - 1;
}
/* lexeme_no is the number of elements processed (ie N) */
lexeme_no++;
/* We prune the D structure after processing each bucket */
if (lexeme_no % bucket_width == 0)
{
prune_lexemes_hashtable(lexemes_tab, b_current);
b_current++;
}
/* Advance to the next WordEntry in the tsvector */
curentryptr++;
}
}
/* We can only compute real stats if we found some non-null values. */
if (null_cnt < samplerows)
{
int nonnull_cnt = samplerows - null_cnt;
int i;
TrackItem **sort_table;
int track_len;
int cutoff_freq;
int minfreq,
maxfreq;
stats->stats_valid = true;
/* Do the simple null-frac and average width stats */
stats->stanullfrac = (double) null_cnt / (double) samplerows;
stats->stawidth = total_width / (double) nonnull_cnt;
/* Assume it's a unique column (see notes above) */
stats->stadistinct = -1.0;
/*
* Construct an array of the interesting hashtable items, that is,
* those meeting the cutoff frequency (s - epsilon)*N. Also identify
* the minimum and maximum frequencies among these items.
*
* Since epsilon = s/10 and bucket_width = 1/epsilon, the cutoff
* frequency is 9*N / bucket_width.
*/
cutoff_freq = 9 * lexeme_no / bucket_width;
i = hash_get_num_entries(lexemes_tab); /* surely enough space */
sort_table = (TrackItem **) palloc(sizeof(TrackItem *) * i);
hash_seq_init(&scan_status, lexemes_tab);
track_len = 0;
minfreq = lexeme_no;
maxfreq = 0;
while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
{
if (item->frequency > cutoff_freq)
{
sort_table[track_len++] = item;
minfreq = Min(minfreq, item->frequency);
maxfreq = Max(maxfreq, item->frequency);
}
}
Assert(track_len <= i);
/* emit some statistics for debug purposes */
elog(DEBUG3, "tsvector_stats: target # mces = %d, bucket width = %d, "
"# lexemes = %d, hashtable size = %d, usable entries = %d",
num_mcelem, bucket_width, lexeme_no, i, track_len);
/*
* If we obtained more lexemes than we really want, get rid of those
* with least frequencies. The easiest way is to qsort the array into
* descending frequency order and truncate the array.
*/
if (num_mcelem < track_len)
{
qsort(sort_table, track_len, sizeof(TrackItem *),
trackitem_compare_frequencies_desc);
/* reset minfreq to the smallest frequency we're keeping */
minfreq = sort_table[num_mcelem - 1]->frequency;
}
else
num_mcelem = track_len;
/* Generate MCELEM slot entry */
if (num_mcelem > 0)
{
MemoryContext old_context;
Datum *mcelem_values;
float4 *mcelem_freqs;
/*
* We want to store statistics sorted on the lexeme value using
* first length, then byte-for-byte comparison. The reason for
* doing length comparison first is that we don't care about the
* ordering so long as it's consistent, and comparing lengths
* first gives us a chance to avoid a strncmp() call.
*
* This is different from what we do with scalar statistics --
* they get sorted on frequencies. The rationale is that we
* usually search through most common elements looking for a
* specific value, so we can grab its frequency. When values are
* presorted we can employ binary search for that. See
* ts_selfuncs.c for a real usage scenario.
*/
qsort(sort_table, num_mcelem, sizeof(TrackItem *),
trackitem_compare_lexemes);
/* Must copy the target values into anl_context */
old_context = MemoryContextSwitchTo(stats->anl_context);
/*
* We sorted statistics on the lexeme value, but we want to be
* able to find out the minimal and maximal frequency without
* going through all the values. We keep those two extra
* frequencies in two extra cells in mcelem_freqs.
*
* (Note: the MCELEM statistics slot definition allows for a third
* extra number containing the frequency of nulls, but we don't
* create that for a tsvector column, since null elements aren't
* possible.)
*/
mcelem_values = (Datum *) palloc(num_mcelem * sizeof(Datum));
mcelem_freqs = (float4 *) palloc((num_mcelem + 2) * sizeof(float4));
/*
* See comments above about use of nonnull_cnt as the divisor for
* the final frequency estimates.
*/
for (i = 0; i < num_mcelem; i++)
{
TrackItem *item = sort_table[i];
mcelem_values[i] =
PointerGetDatum(cstring_to_text_with_len(item->key.lexeme,
item->key.length));
mcelem_freqs[i] = (double) item->frequency / (double) nonnull_cnt;
}
mcelem_freqs[i++] = (double) minfreq / (double) nonnull_cnt;
mcelem_freqs[i] = (double) maxfreq / (double) nonnull_cnt;
MemoryContextSwitchTo(old_context);
stats->stakind[0] = STATISTIC_KIND_MCELEM;
stats->staop[0] = TextEqualOperator;
stats->stanumbers[0] = mcelem_freqs;
/* See above comment about two extra frequency fields */
stats->numnumbers[0] = num_mcelem + 2;
stats->stavalues[0] = mcelem_values;
stats->numvalues[0] = num_mcelem;
/* We are storing text values */
stats->statypid[0] = TEXTOID;
stats->statyplen[0] = -1; /* typlen, -1 for varlena */
stats->statypbyval[0] = false;
stats->statypalign[0] = 'i';
}
}
else
{
/* We found only nulls; assume the column is entirely null */
stats->stats_valid = true;
stats->stanullfrac = 1.0;
stats->stawidth = 0; /* "unknown" */
stats->stadistinct = 0.0; /* "unknown" */
}
/*
* We don't need to bother cleaning up any of our temporary palloc's. The
* hashtable should also go away, as it used a child memory context.
*/
}
| static int lexeme_compare | ( | const void * | key1, | |
| const void * | key2 | |||
| ) | [static] |
Definition at line 484 of file ts_typanalyze.c.
References LexemeHashKey::length, and LexemeHashKey::lexeme.
Referenced by lexeme_match(), and trackitem_compare_lexemes().
{
const LexemeHashKey *d1 = (const LexemeHashKey *) key1;
const LexemeHashKey *d2 = (const LexemeHashKey *) key2;
/* First, compare by length */
if (d1->length > d2->length)
return 1;
else if (d1->length < d2->length)
return -1;
/* Lengths are equal, do a byte-by-byte comparison */
return strncmp(d1->lexeme, d2->lexeme, d1->length);
}
Definition at line 462 of file ts_typanalyze.c.
References DatumGetUInt32, hash_any(), LexemeHashKey::length, and LexemeHashKey::lexeme.
{
const LexemeHashKey *l = (const LexemeHashKey *) key;
return DatumGetUInt32(hash_any((const unsigned char *) l->lexeme,
l->length));
}
| static int lexeme_match | ( | const void * | key1, | |
| const void * | key2, | |||
| Size | keysize | |||
| ) | [static] |
Definition at line 474 of file ts_typanalyze.c.
References lexeme_compare().
{
/* The keysize parameter is superfluous, the keys store their lengths */
return lexeme_compare(key1, key2);
}
| static void prune_lexemes_hashtable | ( | HTAB * | lexemes_tab, | |
| int | b_current | |||
| ) | [static] |
Definition at line 440 of file ts_typanalyze.c.
References TrackItem::delta, elog, ERROR, TrackItem::frequency, HASH_REMOVE, hash_search(), hash_seq_init(), hash_seq_search(), TrackItem::key, and NULL.
Referenced by compute_tsvector_stats().
{
HASH_SEQ_STATUS scan_status;
TrackItem *item;
hash_seq_init(&scan_status, lexemes_tab);
while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
{
if (item->frequency + item->delta <= b_current)
{
if (hash_search(lexemes_tab, (const void *) &item->key,
HASH_REMOVE, NULL) == NULL)
elog(ERROR, "hash table corrupted");
}
}
}
| static int trackitem_compare_frequencies_desc | ( | const void * | e1, | |
| const void * | e2 | |||
| ) | [static] |
Definition at line 502 of file ts_typanalyze.c.
Referenced by compute_tsvector_stats().
| static int trackitem_compare_lexemes | ( | const void * | e1, | |
| const void * | e2 | |||
| ) | [static] |
Definition at line 514 of file ts_typanalyze.c.
References lexeme_compare().
Referenced by compute_tsvector_stats().
{
const TrackItem *const * t1 = (const TrackItem *const *) e1;
const TrackItem *const * t2 = (const TrackItem *const *) e2;
return lexeme_compare(&(*t1)->key, &(*t2)->key);
}
| Datum ts_typanalyze | ( | PG_FUNCTION_ARGS | ) |
Definition at line 54 of file ts_typanalyze.c.
References VacAttrStats::attr, VacAttrStats::compute_stats, default_statistics_target, VacAttrStats::minrows, PG_GETARG_POINTER, and PG_RETURN_BOOL.
{
VacAttrStats *stats = (VacAttrStats *) PG_GETARG_POINTER(0);
Form_pg_attribute attr = stats->attr;
/* If the attstattarget column is negative, use the default value */
/* NB: it is okay to scribble on stats->attr since it's a copy */
if (attr->attstattarget < 0)
attr->attstattarget = default_statistics_target;
stats->compute_stats = compute_tsvector_stats;
/* see comment about the choice of minrows in commands/analyze.c */
stats->minrows = 300 * attr->attstattarget;
PG_RETURN_BOOL(true);
}
1.7.1