[Summary view] [Print] [Text view]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  /**
  18   * A class for loading and preparing grade data from import.
  19   *
  20   * @package   gradeimport_csv
  21   * @copyright 2014 Adrian Greeve <[email protected]>
  22   * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  23   */
  24  
  25  defined('MOODLE_INTERNAL') || die();
  26  
  27  /**
  28   * A class for loading and preparing grade data from import.
  29   *
  30   * @package   gradeimport_csv
  31   * @copyright 2014 Adrian Greeve <[email protected]>
  32   * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  33   */
  34  class gradeimport_csv_load_data {
  35  
  36      /** @var string $error csv import error. */
  37      protected $error;
  38      /** @var int $iid Unique identifier for these csv records. */
  39      protected $iid;
  40      /** @var array $headers Column names for the data. */
  41      protected $headers;
  42      /** @var array $previewdata A subsection of the csv imported data. */
  43      protected $previewdata;
  44  
  45      // The map_user_data_with_value variables.
  46      /** @var array $newgrades Grades to be inserted into the gradebook. */
  47      protected $newgrades;
  48      /** @var array $newfeedbacks Feedback to be inserted into the gradebook. */
  49      protected $newfeedbacks;
  50      /** @var int $studentid Student ID*/
  51      protected $studentid;
  52  
  53      // The prepare_import_grade_data() variables.
  54      /** @var bool $status The current status of the import. True = okay, False = errors. */
  55      protected $status;
  56      /** @var int $importcode The code for this batch insert. */
  57      protected $importcode;
  58      /** @var array $gradebookerrors An array of errors from trying to import into the gradebook. */
  59      protected $gradebookerrors;
  60      /** @var array $newgradeitems An array of new grade items to be inserted into the gradebook. */
  61      protected $newgradeitems;
  62  
  63      /**
  64       * Load CSV content for previewing.
  65       *
  66       * @param string $text The grade data being imported.
  67       * @param string $encoding The type of encoding the file uses.
  68       * @param string $separator The separator being used to define each field.
  69       * @param int $previewrows How many rows are being previewed.
  70       */
  71      public function load_csv_content($text, $encoding, $separator, $previewrows) {
  72          $this->raise_limits();
  73  
  74          $this->iid = csv_import_reader::get_new_iid('grade');
  75          $csvimport = new csv_import_reader($this->iid, 'grade');
  76  
  77          $csvimport->load_csv_content($text, $encoding, $separator);
  78          $this->error = $csvimport->get_error();
  79  
  80          // If there are no import errors then proceed.
  81          if (empty($this->error)) {
  82  
  83              // Get header (field names).
  84              $this->headers = $csvimport->get_columns();
  85              $this->trim_headers();
  86  
  87              $csvimport->init();
  88              $this->previewdata = array();
  89  
  90              for ($numlines = 0; $numlines <= $previewrows; $numlines++) {
  91                  $lines = $csvimport->next();
  92                  if ($lines) {
  93                      $this->previewdata[] = $lines;
  94                  }
  95              }
  96          }
  97      }
  98  
  99      /**
 100       * Gets all of the grade items in this course.
 101       *
 102       * @param int $courseid Course id;
 103       * @return array An array of grade items for the course.
 104       */
 105      public static function fetch_grade_items($courseid) {
 106          $gradeitems = null;
 107          if ($allgradeitems = grade_item::fetch_all(array('courseid' => $courseid))) {
 108              foreach ($allgradeitems as $gradeitem) {
 109                  // Skip course type and category type.
 110                  if ($gradeitem->itemtype == 'course' || $gradeitem->itemtype == 'category') {
 111                      continue;
 112                  }
 113  
 114                  $displaystring = null;
 115                  if (!empty($gradeitem->itemmodule)) {
 116                      $displaystring = get_string('modulename', $gradeitem->itemmodule).get_string('labelsep', 'langconfig')
 117                              .$gradeitem->get_name();
 118                  } else {
 119                      $displaystring = $gradeitem->get_name();
 120                  }
 121                  $gradeitems[$gradeitem->id] = $displaystring;
 122              }
 123          }
 124          return $gradeitems;
 125      }
 126  
 127      /**
 128       * Cleans the column headers from the CSV file.
 129       */
 130      protected function trim_headers() {
 131          foreach ($this->headers as $i => $h) {
 132              $h = trim($h); // Remove whitespace.
 133              $h = clean_param($h, PARAM_RAW); // Clean the header.
 134              $this->headers[$i] = $h;
 135          }
 136      }
 137  
 138      /**
 139       * Raises the php execution time and memory limits for importing the CSV file.
 140       */
 141      protected function raise_limits() {
 142          // Large files are likely to take their time and memory. Let PHP know
 143          // that we'll take longer, and that the process should be recycled soon
 144          // to free up memory.
 145          core_php_time_limit::raise();
 146          raise_memory_limit(MEMORY_EXTRA);
 147      }
 148  
 149      /**
 150       * Inserts a record into the grade_import_values table. This also adds common record information.
 151       *
 152       * @param object $record The grade record being inserted into the database.
 153       * @param int $studentid The student ID.
 154       * @return bool|int true or insert id on success. Null if the grade value is too high.
 155       */
 156      protected function insert_grade_record($record, $studentid) {
 157          global $DB, $USER, $CFG;
 158          $record->importcode = $this->importcode;
 159          $record->userid     = $studentid;
 160          $record->importer   = $USER->id;
 161          // By default the maximum grade is 100.
 162          $gradepointmaximum = 100;
 163          // If the grade limit has been increased then use the gradepointmax setting.
 164          if ($CFG->unlimitedgrades) {
 165              $gradepointmaximum = $CFG->gradepointmax;
 166          }
 167          // If the record final grade is set then check that the grade value isn't too high.
 168          // Final grade will not be set if we are inserting feedback.
 169          if (!isset($record->finalgrade) || $record->finalgrade <= $gradepointmaximum) {
 170              return $DB->insert_record('grade_import_values', $record);
 171          } else {
 172              $this->cleanup_import(get_string('gradevaluetoobig', 'grades', $gradepointmaximum));
 173              return null;
 174          }
 175      }
 176  
 177      /**
 178       * Insert the new grade into the grade item buffer table.
 179       *
 180       * @param array $header The column headers from the CSV file.
 181       * @param int $key Current row identifier.
 182       * @param string $value The value for this row (final grade).
 183       * @return array new grades that are ready for commiting to the gradebook.
 184       */
 185      protected function import_new_grade_item($header, $key, $value) {
 186          global $DB, $USER;
 187  
 188          // First check if header is already in temp database.
 189          if (empty($this->newgradeitems[$key])) {
 190  
 191              $newgradeitem = new stdClass();
 192              $newgradeitem->itemname = $header[$key];
 193              $newgradeitem->importcode = $this->importcode;
 194              $newgradeitem->importer = $USER->id;
 195  
 196              // Insert into new grade item buffer.
 197              $this->newgradeitems[$key] = $DB->insert_record('grade_import_newitem', $newgradeitem);
 198          }
 199          $newgrade = new stdClass();
 200          $newgrade->newgradeitem = $this->newgradeitems[$key];
 201  
 202          // If the user has a grade for this grade item.
 203          if (trim($value) != '-') {
 204              // Instead of omitting the grade we could insert one with finalgrade set to 0.
 205              // We do not have access to grade item min grade.
 206              $newgrade->finalgrade = $value;
 207              $newgrades[] = $newgrade;
 208          }
 209          return $newgrades;
 210      }
 211  
 212      /**
 213       * Check that the user is in the system.
 214       *
 215       * @param string $value The value, from the csv file, being mapped to identify the user.
 216       * @param array $userfields Contains the field and label being mapped from.
 217       * @return int Returns the user ID if it exists, otherwise null.
 218       */
 219      protected function check_user_exists($value, $userfields) {
 220          global $DB;
 221  
 222          $usercheckproblem = false;
 223          $user = null;
 224          // The user may use the incorrect field to match the user. This could result in an exception.
 225          try {
 226              $user = $DB->get_record('user', array($userfields['field'] => $value));
 227          } catch (Exception $e) {
 228              $usercheckproblem = true;
 229          }
 230          // Field may be fine, but no records were returned.
 231          if (!$user || $usercheckproblem) {
 232              $usermappingerrorobj = new stdClass();
 233              $usermappingerrorobj->field = $userfields['label'];
 234              $usermappingerrorobj->value = $value;
 235              $this->cleanup_import(get_string('usermappingerror', 'grades', $usermappingerrorobj));
 236              unset($usermappingerrorobj);
 237              return null;
 238          }
 239          return $user->id;
 240      }
 241  
 242      /**
 243       * Check to see if the feedback matches a grade item.
 244       *
 245       * @param int $courseid The course ID.
 246       * @param int $itemid The ID of the grade item that the feedback relates to.
 247       * @param string $value The actual feedback being imported.
 248       * @return object Creates a feedback object with the item ID and the feedback value.
 249       */
 250      protected function create_feedback($courseid, $itemid, $value) {
 251          // Case of an id, only maps id of a grade_item.
 252          // This was idnumber.
 253          if (!new grade_item(array('id' => $itemid, 'courseid' => $courseid))) {
 254              // Supplied bad mapping, should not be possible since user
 255              // had to pick mapping.
 256              $this->cleanup_import(get_string('importfailed', 'grades'));
 257              return null;
 258          }
 259  
 260          // The itemid is the id of the grade item.
 261          $feedback = new stdClass();
 262          $feedback->itemid   = $itemid;
 263          $feedback->feedback = $value;
 264          return $feedback;
 265      }
 266  
 267      /**
 268       * This updates existing grade items.
 269       *
 270       * @param int $courseid The course ID.
 271       * @param array $map Mapping information provided by the user.
 272       * @param int $key The line that we are currently working on.
 273       * @param bool $verbosescales Form setting for grading with scales.
 274       * @param string $value The grade value.
 275       * @return array grades to be updated.
 276       */
 277      protected function update_grade_item($courseid, $map, $key, $verbosescales, $value) {
 278          // Case of an id, only maps id of a grade_item.
 279          // This was idnumber.
 280          if (!$gradeitem = new grade_item(array('id' => $map[$key], 'courseid' => $courseid))) {
 281              // Supplied bad mapping, should not be possible since user
 282              // had to pick mapping.
 283              $this->cleanup_import(get_string('importfailed', 'grades'));
 284              return null;
 285          }
 286  
 287          // Check if grade item is locked if so, abort.
 288          if ($gradeitem->is_locked()) {
 289              $this->cleanup_import(get_string('gradeitemlocked', 'grades'));
 290              return null;
 291          }
 292  
 293          $newgrade = new stdClass();
 294          $newgrade->itemid = $gradeitem->id;
 295          if ($gradeitem->gradetype == GRADE_TYPE_SCALE and $verbosescales) {
 296              if ($value === '' or $value == '-') {
 297                  $value = null; // No grade.
 298              } else {
 299                  $scale = $gradeitem->load_scale();
 300                  $scales = explode(',', $scale->scale);
 301                  $scales = array_map('trim', $scales); // Hack - trim whitespace around scale options.
 302                  array_unshift($scales, '-'); // Scales start at key 1.
 303                  $key = array_search($value, $scales);
 304                  if ($key === false) {
 305                      $this->cleanup_import(get_string('badgrade', 'grades'));
 306                      return null;
 307                  }
 308                  $value = $key;
 309              }
 310              $newgrade->finalgrade = $value;
 311          } else {
 312              if ($value === '' or $value == '-') {
 313                  $value = null; // No grade.
 314              } else {
 315                  // If the value has a local decimal or can correctly be unformatted, do it.
 316                  $validvalue = unformat_float($value, true);
 317                  if ($validvalue !== false) {
 318                      $value = $validvalue;
 319                  } else {
 320                      // Non numeric grade value supplied, possibly mapped wrong column.
 321                      $this->cleanup_import(get_string('badgrade', 'grades'));
 322                      return null;
 323                  }
 324              }
 325              $newgrade->finalgrade = $value;
 326          }
 327          $this->newgrades[] = $newgrade;
 328          return $this->newgrades;
 329      }
 330  
 331      /**
 332       * Clean up failed CSV grade import. Clears the temp table for inserting grades.
 333       *
 334       * @param string $notification The error message to display from the unsuccessful grade import.
 335       */
 336      protected function cleanup_import($notification) {
 337          $this->status = false;
 338          import_cleanup($this->importcode);
 339          $this->gradebookerrors[] = $notification;
 340      }
 341  
 342      /**
 343       * Check user mapping.
 344       *
 345       * @param string $mappingidentifier The user field that we are matching together.
 346       * @param string $value The value we are checking / importing.
 347       * @param array $header The column headers of the csv file.
 348       * @param array $map Mapping information provided by the user.
 349       * @param int $key Current row identifier.
 350       * @param int $courseid The course ID.
 351       * @param int $feedbackgradeid The ID of the grade item that the feedback relates to.
 352       * @param bool $verbosescales Form setting for grading with scales.
 353       */
 354      protected function map_user_data_with_value($mappingidentifier, $value, $header, $map, $key, $courseid, $feedbackgradeid,
 355              $verbosescales) {
 356  
 357          // Fields that the user can be mapped from.
 358          $userfields = array(
 359              'userid' => array(
 360                  'field' => 'id',
 361                  'label' => 'id',
 362              ),
 363              'useridnumber' => array(
 364                  'field' => 'idnumber',
 365                  'label' => 'idnumber',
 366              ),
 367              'useremail' => array(
 368                  'field' => 'email',
 369                  'label' => 'email address',
 370              ),
 371              'username' => array(
 372                  'field' => 'username',
 373                  'label' => 'username',
 374              ),
 375          );
 376  
 377          switch ($mappingidentifier) {
 378              case 'userid':
 379              case 'useridnumber':
 380              case 'useremail':
 381              case 'username':
 382                  // Skip invalid row with blank user field.
 383                  if (!empty($value)) {
 384                      $this->studentid = $this->check_user_exists($value, $userfields[$mappingidentifier]);
 385                  }
 386              break;
 387              case 'new':
 388                  $this->newgrades = $this->import_new_grade_item($header, $key, $value);
 389              break;
 390              case 'feedback':
 391                  if ($feedbackgradeid) {
 392                      $feedback = $this->create_feedback($courseid, $feedbackgradeid, $value);
 393                      if (isset($feedback)) {
 394                          $this->newfeedbacks[] = $feedback;
 395                      }
 396                  }
 397              break;
 398              default:
 399                  // Existing grade items.
 400                  if (!empty($map[$key])) {
 401                      $this->newgrades = $this->update_grade_item($courseid, $map, $key, $verbosescales, $value,
 402                              $mappingidentifier);
 403                  }
 404                  // Otherwise, we ignore this column altogether because user has chosen
 405                  // to ignore them (e.g. institution, address etc).
 406              break;
 407          }
 408      }
 409  
 410      /**
 411       * Checks and prepares grade data for inserting into the gradebook.
 412       *
 413       * @param array $header Column headers of the CSV file.
 414       * @param object $formdata Mapping information from the preview page.
 415       * @param object $csvimport csv import reader object for iterating over the imported CSV file.
 416       * @param int $courseid The course ID.
 417       * @param bool $separatemode If we have groups are they separate?
 418       * @param mixed $currentgroup current group information.
 419       * @param bool $verbosescales Form setting for grading with scales.
 420       * @return bool True if the status for importing is okay, false if there are errors.
 421       */
 422      public function prepare_import_grade_data($header, $formdata, $csvimport, $courseid, $separatemode, $currentgroup,
 423              $verbosescales) {
 424          global $DB, $USER;
 425  
 426          // The import code is used for inserting data into the grade tables.
 427          $this->importcode = $formdata->importcode;
 428          $this->status = true;
 429          $this->headers = $header;
 430          $this->studentid = null;
 431          $this->gradebookerrors = null;
 432          $forceimport = $formdata->forceimport;
 433          // Temporary array to keep track of what new headers are processed.
 434          $this->newgradeitems = array();
 435          $this->trim_headers();
 436          $timeexportkey = null;
 437          $map = array();
 438          // Loops mapping_0, mapping_1 .. mapping_n and construct $map array.
 439          foreach ($header as $i => $head) {
 440              if (isset($formdata->{'mapping_'.$i})) {
 441                  $map[$i] = $formdata->{'mapping_'.$i};
 442              }
 443              if ($head == get_string('timeexported', 'gradeexport_txt')) {
 444                  $timeexportkey = $i;
 445              }
 446          }
 447  
 448          // If mapping information is supplied.
 449          $map[clean_param($formdata->mapfrom, PARAM_RAW)] = clean_param($formdata->mapto, PARAM_RAW);
 450  
 451          // Check for mapto collisions.
 452          $maperrors = array();
 453          foreach ($map as $i => $j) {
 454              if ($j == 0) {
 455                  // You can have multiple ignores.
 456                  continue;
 457              } else {
 458                  if (!isset($maperrors[$j])) {
 459                      $maperrors[$j] = true;
 460                  } else {
 461                      // Collision.
 462                      print_error('cannotmapfield', '', '', $j);
 463                  }
 464              }
 465          }
 466  
 467          $this->raise_limits();
 468  
 469          $csvimport->init();
 470  
 471          while ($line = $csvimport->next()) {
 472              if (count($line) <= 1) {
 473                  // There is no data on this line, move on.
 474                  continue;
 475              }
 476  
 477              // Array to hold all grades to be inserted.
 478              $this->newgrades = array();
 479              // Array to hold all feedback.
 480              $this->newfeedbacks = array();
 481              // Each line is a student record.
 482              foreach ($line as $key => $value) {
 483  
 484                  $value = clean_param($value, PARAM_RAW);
 485                  $value = trim($value);
 486  
 487                  /*
 488                   * the options are
 489                   * 1) userid, useridnumber, usermail, username - used to identify user row
 490                   * 2) new - new grade item
 491                   * 3) id - id of the old grade item to map onto
 492                   * 3) feedback_id - feedback for grade item id
 493                   */
 494  
 495                  // Explode the mapping for feedback into a label 'feedback' and the identifying number.
 496                  $mappingbase = explode("_", $map[$key]);
 497                  $mappingidentifier = $mappingbase[0];
 498                  // Set the feedback identifier if it exists.
 499                  if (isset($mappingbase[1])) {
 500                      $feedbackgradeid = (int)$mappingbase[1];
 501                  } else {
 502                      $feedbackgradeid = '';
 503                  }
 504  
 505                  $this->map_user_data_with_value($mappingidentifier, $value, $header, $map, $key, $courseid, $feedbackgradeid,
 506                          $verbosescales);
 507                  if ($this->status === false) {
 508                      return $this->status;
 509                  }
 510              }
 511  
 512              // No user mapping supplied at all, or user mapping failed.
 513              if (empty($this->studentid) || !is_numeric($this->studentid)) {
 514                  // User not found, abort whole import.
 515                  $this->cleanup_import(get_string('usermappingerrorusernotfound', 'grades'));
 516                  break;
 517              }
 518  
 519              if ($separatemode and !groups_is_member($currentgroup, $this->studentid)) {
 520                  // Not allowed to import into this group, abort.
 521                  $this->cleanup_import(get_string('usermappingerrorcurrentgroup', 'grades'));
 522                  break;
 523              }
 524  
 525              // Insert results of this students into buffer.
 526              if ($this->status and !empty($this->newgrades)) {
 527  
 528                  foreach ($this->newgrades as $newgrade) {
 529  
 530                      // Check if grade_grade is locked and if so, abort.
 531                      if (!empty($newgrade->itemid) and $gradegrade = new grade_grade(array('itemid' => $newgrade->itemid,
 532                              'userid' => $this->studentid))) {
 533                          if ($gradegrade->is_locked()) {
 534                              // Individual grade locked.
 535                              $this->cleanup_import(get_string('gradelocked', 'grades'));
 536                              return $this->status;
 537                          }
 538                          // Check if the force import option is disabled and the last exported date column is present.
 539                          if (!$forceimport && !empty($timeexportkey)) {
 540                              $exportedtime = $line[$timeexportkey];
 541                              if (clean_param($exportedtime, PARAM_INT) != $exportedtime || $exportedtime > time() ||
 542                                      $exportedtime < strtotime("-1 year", time())) {
 543                                  // The date is invalid, or in the future, or more than a year old.
 544                                  $this->cleanup_import(get_string('invalidgradeexporteddate', 'grades'));
 545                                  return $this->status;
 546  
 547                              }
 548                              $timemodified = $gradegrade->get_dategraded();
 549                              if (!empty($timemodified) && ($exportedtime < $timemodified)) {
 550                                  // The item was graded after we exported it, we return here not to override it.
 551                                  $user = core_user::get_user($this->studentid);
 552                                  $this->cleanup_import(get_string('gradealreadyupdated', 'grades', fullname($user)));
 553                                  return $this->status;
 554                              }
 555                          }
 556                      }
 557                      $insertid = self::insert_grade_record($newgrade, $this->studentid);
 558                      // Check to see if the insert was successful.
 559                      if (empty($insertid)) {
 560                          return null;
 561                      }
 562                  }
 563              }
 564  
 565              // Updating/inserting all comments here.
 566              if ($this->status and !empty($this->newfeedbacks)) {
 567                  foreach ($this->newfeedbacks as $newfeedback) {
 568                      $sql = "SELECT *
 569                                FROM {grade_import_values}
 570                               WHERE importcode=? AND userid=? AND itemid=? AND importer=?";
 571                      if ($feedback = $DB->get_record_sql($sql, array($this->importcode, $this->studentid, $newfeedback->itemid,
 572                              $USER->id))) {
 573                          $newfeedback->id = $feedback->id;
 574                          $DB->update_record('grade_import_values', $newfeedback);
 575  
 576                      } else {
 577                          // The grade item for this is not updated.
 578                          $insertid = self::insert_grade_record($newfeedback, $this->studentid);
 579                          // Check to see if the insert was successful.
 580                          if (empty($insertid)) {
 581                              return null;
 582                          }
 583                      }
 584                  }
 585              }
 586          }
 587          return $this->status;
 588      }
 589  
 590      /**
 591       * Returns the headers parameter for this class.
 592       *
 593       * @return array returns headers parameter for this class.
 594       */
 595      public function get_headers() {
 596          return $this->headers;
 597      }
 598  
 599      /**
 600       * Returns the error parameter for this class.
 601       *
 602       * @return string returns error parameter for this class.
 603       */
 604      public function get_error() {
 605          return $this->error;
 606      }
 607  
 608      /**
 609       * Returns the iid parameter for this class.
 610       *
 611       * @return int returns iid parameter for this class.
 612       */
 613      public function get_iid() {
 614          return $this->iid;
 615      }
 616  
 617      /**
 618       * Returns the preview_data parameter for this class.
 619       *
 620       * @return array returns previewdata parameter for this class.
 621       */
 622      public function get_previewdata() {
 623          return $this->previewdata;
 624      }
 625  
 626      /**
 627       * Returns the gradebookerrors parameter for this class.
 628       *
 629       * @return array returns gradebookerrors parameter for this class.
 630       */
 631      public function get_gradebookerrors() {
 632          return $this->gradebookerrors;
 633      }
 634  }