4 #include <glib/gi18n.h>
6 #include <goffice/goffice-features.h>
7 #if (GO_VERSION_EPOCH == 0) && (GO_VERSION_MAJOR == 7) && (GO_VERSION_MINOR == 8)
12 #include <goffice/utils/go-glib-extras.h>
15 #include "engine-helpers.h"
19 #include <sys/types.h>
27 G_GNUC_UNUSED
static QofLogModule log_module = GNC_MOD_IMPORT;
29 const int num_date_formats = 5;
30 const gchar* date_format_user[] = {N_(
"y-m-d"),
37 const int num_currency_formats = 3;
38 const gchar* currency_format_user[] = {N_(
"Locale"),
39 N_(
"Period: 123,456.78"),
40 N_(
"Comma: 123.456,78")
44 gchar* gnc_csv_column_type_strs[GNC_CSV_NUM_COL_TYPES] = {N_(
"None"),
58 static StfParseOptions_t* default_parse_options (
void)
60 StfParseOptions_t* options = stf_parse_options_new();
61 stf_parse_options_set_type (options, PARSE_TYPE_CSV);
62 stf_parse_options_csv_set_separators (options,
",", NULL);
73 static time64 parse_date_with_year (
const char* date_str,
int format)
76 struct tm retvalue, test_retvalue;
78 int i, j, mem_length, orig_year = -1, orig_month = -1, orig_day = -1;
87 regmatch_t pmatch[4] = { {0}, {0}, {0}, {0} };
90 const char* regex =
"^ *([0-9]+) *[-/.'] *([0-9]+) *[-/.'] *([0-9]+).*$|^ *([0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]).*$";
93 regcomp (&preg, regex, REG_EXTENDED);
94 regexec (&preg, date_str, 4, pmatch, 0);
98 if (pmatch[0].rm_eo == 0)
102 if (pmatch[1].rm_so == -1)
107 for (i = 0; date_format_user[format][i]; i++)
109 char segment_type = date_format_user[format][i];
111 if (segment_type ==
'y' || segment_type ==
'm' || segment_type ==
'd')
114 switch (segment_type)
142 for (i = 0; date_format_user[format][i]; i++)
144 char segment_type = date_format_user[format][i];
146 if (segment_type ==
'y' || segment_type ==
'm' || segment_type ==
'd')
150 mem_length = pmatch[j].rm_eo - pmatch[j].rm_so;
151 memcpy (date_segment, date_str + pmatch[j].rm_so, mem_length);
152 date_segment[mem_length] =
'\0';
157 switch (segment_type)
160 retvalue.tm_year = atoi (date_segment);
163 if (retvalue.tm_year < 100)
166 if (retvalue.tm_year < 69)
167 retvalue.tm_year += 100;
170 retvalue.tm_year -= 1900;
171 orig_year = retvalue.tm_year;
175 orig_month = retvalue.tm_mon = atoi (date_segment) - 1;
179 orig_day = retvalue.tm_mday = atoi (date_segment);
191 test_retvalue = retvalue;
193 retvalue.tm_isdst = test_retvalue.tm_isdst;
195 if (retvalue.tm_mday == orig_day &&
196 retvalue.tm_mon == orig_month &&
197 retvalue.tm_year == orig_year)
214 static time64 parse_date_without_year (
const char* date_str,
int format)
217 struct tm retvalue, test_retvalue;
219 int i, j, mem_length, orig_year = -1, orig_month = -1, orig_day = -1;
228 regmatch_t pmatch[3] = { {0}, {0}, {0} };
231 const char* regex =
"^ *([0-9]+) *[-/.'] *([0-9]+).*$";
234 regcomp (&preg, regex, REG_EXTENDED);
235 regexec (&preg, date_str, 3, pmatch, 0);
239 if (pmatch[0].rm_eo == 0)
246 orig_year = retvalue.tm_year;
253 for (i = 0; date_format_user[format][i]; i++)
255 char segment_type = date_format_user[format][i];
257 if (segment_type ==
'm' || segment_type ==
'd')
261 mem_length = pmatch[j].rm_eo - pmatch[j].rm_so;
262 date_segment = g_new (gchar, mem_length);
263 memcpy(date_segment, date_str + pmatch[j].rm_so, mem_length);
264 date_segment[mem_length] =
'\0';
269 switch (segment_type)
272 orig_month = retvalue.tm_mon = atoi (date_segment) - 1;
276 orig_day = retvalue.tm_mday = atoi (date_segment);
279 g_free (date_segment);
289 test_retvalue = retvalue;
291 retvalue.tm_isdst = test_retvalue.tm_isdst;
293 if (retvalue.tm_mday == orig_day &&
294 retvalue.tm_mon == orig_month &&
295 retvalue.tm_year == orig_year)
315 if (strchr (date_format_user[format],
'y'))
316 return parse_date_with_year (date_str, format);
318 return parse_date_without_year (date_str, format);
327 parse_data->encoding =
"UTF-8";
337 parse_data->
options = default_parse_options();
340 parse_data->
chunk = g_string_chunk_new(100 * 1024);
359 if (parse_data->
file_str.begin != NULL)
360 g_free (parse_data->
file_str.begin);
363 stf_parse_general_free (parse_data->
orig_lines);
368 if (parse_data->
options != NULL)
369 stf_parse_options_free (parse_data->
options);
384 g_free (transactions->data);
385 transactions = g_list_next (transactions);
387 while (transactions != NULL);
391 g_free (parse_data->
chunk);
406 gsize bytes_read, bytes_written;
412 if (parse_data->
file_str.begin != NULL)
418 "UTF-8", encoding, &bytes_read, &bytes_written,
421 if (parse_data->
file_str.begin == NULL)
427 parse_data->encoding = (gchar*)encoding;
446 const char* guess_enc = NULL;
449 parse_data->
raw_mapping = g_mapped_file_new (filename, FALSE, error);
454 parse_data->
raw_str.begin = NULL;
455 g_clear_error (error);
456 g_set_error (error, 0, GNC_CSV_FILE_OPEN_ERR,
"%s", _(
"File opening failed."));
465 if (!g_mapped_file_get_length (parse_data->
raw_mapping) == 0)
466 guess_enc = go_guess_encoding ((
const char*)(parse_data->
raw_str.begin),
469 if (guess_enc == NULL)
471 g_set_error (error, 0, GNC_CSV_ENCODING_ERR,
"%s", _(
"Unknown encoding."));
477 if (parse_data->
file_str.begin == NULL)
479 g_set_error (error, 0, GNC_CSV_ENCODING_ERR,
"%s", _(
"Unknown encoding."));
505 stf_parse_general_free (parse_data->
orig_lines);
509 if (parse_data->
file_str.begin != NULL)
527 g_array_sized_new (FALSE, FALSE,
sizeof(
int), parse_data->
orig_lines->len);
531 for (i = 0; i < parse_data->
orig_lines->len; i++)
533 int length = ((GPtrArray*)parse_data->
orig_lines->pdata[i])->len;
542 g_set_error (error, 0, 0,
"Parsing failed.");
547 for (i = 0; i < parse_data->
orig_lines->len; i++)
549 if (max_cols < ((GPtrArray*)(parse_data->
orig_lines->pdata[i]))->len)
550 max_cols = ((GPtrArray*)(parse_data->
orig_lines->pdata[i]))->len;
561 parse_data->
column_types = g_array_sized_new (FALSE, FALSE,
sizeof(
int),
629 case GNC_CSV_BALANCE:
630 case GNC_CSV_DEPOSIT:
631 case GNC_CSV_WITHDRAWAL:
632 if (prop->
value != NULL)
646 static gboolean trans_property_set (
TransProperty* prop,
char* str)
648 char *endptr, *possible_currency_symbol, *str_dupe;
659 case GNC_CSV_DESCRIPTION:
662 prop->
value = g_strdup (str);
665 case GNC_CSV_BALANCE:
666 case GNC_CSV_DEPOSIT:
667 case GNC_CSV_WITHDRAWAL:
668 str_dupe = g_strdup (str);
670 reti = regcomp(®ex,
"[0-9]", 0);
671 reti = regexec(®ex, str_dupe, 0, NULL, 0);
672 if (reti == REG_NOMATCH)
675 str_dupe = g_strdup (
"0");
678 for (possible_currency_symbol = str_dupe; *possible_currency_symbol;
679 possible_currency_symbol = g_utf8_next_char (possible_currency_symbol))
681 if (g_unichar_type (g_utf8_get_char (possible_currency_symbol)) == G_UNICODE_CURRENCY_SYMBOL)
686 char *next_symbol = g_utf8_next_char (possible_currency_symbol), *last_symbol = next_symbol;
688 last_symbol = g_utf8_next_char (last_symbol);
694 memmove (possible_currency_symbol, next_symbol, last_symbol - next_symbol + 1);
704 if (!(xaccParseAmount (str_dupe, TRUE, &val, &endptr)))
712 if (!(xaccParseAmountExtended (str_dupe, TRUE,
'-',
'.',
',',
"\003\003",
"$+", &val, &endptr)))
720 if (!(xaccParseAmountExtended (str_dupe, TRUE,
'-',
',',
'.',
"\003\003",
"$+", &val, &endptr)))
764 g_list_free (properties_begin);
773 static void trans_property_list_add (
TransProperty* property)
775 property->list->properties = g_list_append (property->
list->
properties, property);
788 xaccSplitSetAccount (split, account);
789 xaccSplitSetParent (split, trans);
793 gnc_set_num_action (trans, split, num, NULL);
803 static gboolean trans_property_list_verify_essentials (
TransPropertyList* list, gchar** error)
807 enum PossibleErrorTypes {NO_DATE, NO_AMOUNT, NUM_OF_POSSIBLE_ERRORS};
808 gchar* possible_errors[NUM_OF_POSSIBLE_ERRORS] =
810 N_(
"No date column."),
811 N_(
"No balance, deposit, or withdrawal column.")
813 int possible_error_lengths[NUM_OF_POSSIBLE_ERRORS] = {0};
814 GList *properties_begin = list->
properties, *errors_list = NULL;
822 possible_errors[NO_DATE] = NULL;
825 case GNC_CSV_BALANCE:
826 case GNC_CSV_DEPOSIT:
827 case GNC_CSV_WITHDRAWAL:
828 possible_errors[NO_AMOUNT] = NULL;
836 for (i = 0; i < NUM_OF_POSSIBLE_ERRORS; i++)
838 if (possible_errors[i] != NULL)
840 errors_list = g_list_append (errors_list, GINT_TO_POINTER(i));
843 possible_error_lengths[i] = strlen (_(possible_errors[i]));
848 if (errors_list == NULL)
853 int full_error_size = 0, string_length = 0;
854 GList* errors_list_begin = errors_list;
855 gchar *error_message, *error_message_begin;
861 full_error_size += possible_error_lengths[GPOINTER_TO_INT(errors_list->data)] + 1;
862 errors_list = g_list_next (errors_list);
864 errors_list = errors_list_begin;
867 error_message = error_message_begin = g_new (gchar, full_error_size);
870 i = GPOINTER_TO_INT(errors_list->data);
871 string_length = possible_error_lengths[i];
874 strncpy(error_message, _(possible_errors[i]), string_length);
875 error_message += string_length;
876 *error_message =
' ';
879 errors_list = g_list_next (errors_list);
881 *error_message =
'\0';
882 g_list_free (errors_list_begin);
884 *error = error_message_begin;
905 gboolean amount_set = FALSE;
910 trans_line->
num = NULL;
915 trans_line->line_no = -1;
918 if (!trans_property_list_verify_essentials (list, error))
939 case GNC_CSV_DESCRIPTION:
951 num = g_strdup ((
char*)(prop->
value));
955 trans_line->
num = g_strdup ((
char*)(prop->
value));
958 case GNC_CSV_DEPOSIT:
959 if (prop->
value != NULL)
971 case GNC_CSV_WITHDRAWAL:
972 if (prop->
value != NULL)
984 case GNC_CSV_BALANCE:
986 if (!amount_set && prop->
value != NULL)
998 trans_add_split (trans_line->trans, list->
account, book, amount, num);
1016 gboolean redo_errors)
1018 gboolean hasBalanceColumn;
1019 int i, j, max_cols = 0;
1021 GList *error_lines = NULL, *begin_error_lines = NULL;
1025 GList* last_transaction = NULL;
1031 begin_error_lines = error_lines = parse_data->
error_lines;
1050 last_transaction = NULL;
1056 while (g_list_next (last_transaction) != NULL)
1058 last_transaction = g_list_next (last_transaction);
1062 if (error_lines == NULL)
1065 i = GPOINTER_TO_INT(error_lines->data);
1072 last_transaction = NULL;
1079 while (i < parse_data->end_row)
1081 GPtrArray* line = parse_data->
orig_lines->pdata[i];
1083 gboolean errors = FALSE;
1084 gchar* error_message = NULL;
1088 for (j = 0; j < line->len; j++)
1091 if ((column_types->data[j] != GNC_CSV_NONE) && (column_types->data[j] != GNC_CSV_ACCOUNT))
1094 TransProperty*
property = trans_property_new (column_types->data[j], list);
1095 gboolean succeeded = trans_property_set (property, line->pdata[j]);
1100 trans_property_list_add (property);
1105 error_message = g_strdup_printf (_(
"%s column could not be understood."),
1106 _(gnc_csv_column_type_strs[property->type]));
1107 trans_property_free (property);
1116 trans_line = trans_property_list_to_trans (list, &error_message);
1117 errors = trans_line == NULL;
1120 trans_property_list_free (list);
1126 GINT_TO_POINTER(i));
1130 g_free(line->pdata[line->len - 1]);
1131 line->pdata[line->len - 1] = error_message;
1136 g_ptr_array_add (line, error_message);
1142 trans_line->line_no = i;
1150 if (last_transaction == NULL ||
1155 if (last_transaction == NULL)
1158 last_transaction = g_list_next (last_transaction);
1163 GList* insertion_spot = last_transaction;
1164 while (insertion_spot != NULL &&
1167 insertion_spot = g_list_previous (insertion_spot);
1171 if (insertion_spot == NULL)
1174 insertion_spot = g_list_next (insertion_spot);
1184 error_lines = g_list_next (error_lines);
1185 if (error_lines == NULL)
1188 i = GPOINTER_TO_INT(error_lines->data);
1200 hasBalanceColumn = FALSE;
1203 if (parse_data->
column_types->data[i] == GNC_CSV_BALANCE)
1205 hasBalanceColumn = TRUE;
1210 if (hasBalanceColumn)
1221 while (transactions != NULL)
1242 SplitList* next_splits = g_list_next (splits);
1244 splits = next_splits;
1247 trans_add_split (trans_line->trans, account,
1248 gnc_account_get_book (account), amount, trans_line->
num);
1249 if (trans_line->
num)
1250 g_free (trans_line->
num);
1258 transactions = g_list_next (transactions);
1264 g_list_free (begin_error_lines);
1268 for (i = 0; i < parse_data->
orig_lines->len; i++)
1270 if (max_cols < ((GPtrArray*)(parse_data->
orig_lines->pdata[i]))->len)
1271 max_cols = ((GPtrArray*)(parse_data->
orig_lines->pdata[i]))->len;
1275 for (; i < max_cols; i++)
void xaccSplitSetValue(Split *s, gnc_numeric amt)
GMappedFile * raw_mapping
Transaction * xaccMallocTransaction(QofBook *book)
void xaccTransSetDatePostedSecsNormalized(Transaction *trans, time64 time)
time64 xaccTransGetDate(const Transaction *trans)
gnc_numeric double_to_gnc_numeric(double n, gint64 denom, gint how)
int gnc_csv_convert_encoding(GncCsvParseData *parse_data, const char *encoding, GError **error)
utility functions for the GnuCash UI
gboolean xaccSplitDestroy(Split *split)
int xaccAccountGetCommoditySCU(const Account *acc)
gnc_numeric gnc_numeric_neg(gnc_numeric a)
void xaccTransSetNotes(Transaction *trans, const char *notes)
int gnc_csv_parse_to_trans(GncCsvParseData *parse_data, Account *account, gboolean redo_errors)
void xaccTransSetDescription(Transaction *trans, const char *desc)
gnc_numeric gnc_numeric_add(gnc_numeric a, gnc_numeric b, gint64 denom, gint how)
GArray * orig_row_lengths
time64 parse_date(const char *date_str, int format)
struct tm * gnc_localtime_r(const time64 *secs, struct tm *time)
fill out a time struct from a 64-bit time value adjusted for the current time zone.
void xaccTransSetCurrency(Transaction *trans, gnc_commodity *curr)
void gnc_csv_parse_data_free(GncCsvParseData *parse_data)
void xaccSplitSetAmount(Split *s, gnc_numeric amt)
StfParseOptions_t * options
GncCsvParseData * gnc_csv_new_parse_data(void)
time64 gnc_mktime(struct tm *time)
calculate seconds from the epoch given a time struct
void xaccTransBeginEdit(Transaction *trans)
gnc_numeric xaccAccountGetBalanceAsOfDate(Account *acc, time64 date)
int gnc_csv_parse(GncCsvParseData *parse_data, gboolean guessColTypes, GError **error)
Split * xaccMallocSplit(QofBook *book)
gnc_numeric gnc_numeric_sub(gnc_numeric a, gnc_numeric b, gint64 denom, gint how)
gnc_commodity * xaccAccountGetCommodity(const Account *acc)
time64 gnc_time(time64 *tbuf)
get the current local time
int gnc_csv_load_file(GncCsvParseData *parse_data, const char *filename, GError **error)
SplitList * xaccTransGetSplitList(const Transaction *trans)
const gchar * QofLogModule