Grow.c

   1 /*
   2  * mdadm - manage Linux "md" devices aka RAID arrays.
   3  *
   4  * Copyright (C) 2001-2013 Neil Brown <neilb@suse.de>
   5  *
   6  *
   7  *    This program is free software; you can redistribute it and/or modify
   8  *    it under the terms of the GNU General Public License as published by
   9  *    the Free Software Foundation; either version 2 of the License, or
  10  *    (at your option) any later version.
  11  *
  12  *    This program is distributed in the hope that it will be useful,
  13  *    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *    GNU General Public License for more details.
  16  *
  17  *    You should have received a copy of the GNU General Public License
  18  *    along with this program; if not, write to the Free Software
  19  *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20  *
  21  *    Author: Neil Brown
  22  *    Email: <neilb@suse.de>
  23  */
  24 #include        "mdadm.h"
  25 #include        "dlink.h"
  26 #include        <sys/mman.h>
  27 #include        <stddef.h>
  28 #include        <stdint.h>
  29 #include        <signal.h>
  30 #include        <sys/wait.h>
  31
  32 #if ! defined(__BIG_ENDIAN) && ! defined(__LITTLE_ENDIAN)
  33 #error no endian defined
  34 #endif
  35 #include        "md_u.h"
  36 #include        "md_p.h"
  37
  38 int restore_backup(struct supertype *st,
  39                    struct mdinfo *content,
  40                    int working_disks,
  41                    int next_spare,
  42                    char **backup_filep,
  43                    int verbose)
  44 {
  45         int i;
  46         int *fdlist;
  47         struct mdinfo *dev;
  48         int err;
  49         int disk_count = next_spare + working_disks;
  50         char *backup_file = *backup_filep;
  51
  52         dprintf("Called restore_backup()\n");
  53         fdlist = xmalloc(sizeof(int) * disk_count);
  54
  55         enable_fds(next_spare);
  56         for (i = 0; i < next_spare; i++)
  57                 fdlist[i] = -1;
  58         for (dev = content->devs; dev; dev = dev->next) {
  59                 char buf[22];
  60                 int fd;
  61                 sprintf(buf, "%d:%d",
  62                         dev->disk.major,
  63                         dev->disk.minor);
  64                 fd = dev_open(buf, O_RDWR);
  65
  66                 if (dev->disk.raid_disk >= 0)
  67                         fdlist[dev->disk.raid_disk] = fd;
  68                 else
  69                         fdlist[next_spare++] = fd;
  70         }
  71
  72         if (!backup_file) {
  73                 backup_file = locate_backup(content->sys_name);
  74                 *backup_filep = backup_file;
  75         }
  76
  77         if (st->ss->external && st->ss->recover_backup)
  78                 err = st->ss->recover_backup(st, content);
  79         else
  80                 err = Grow_restart(st, content, fdlist, next_spare,
  81                                    backup_file, verbose > 0);
  82
  83         while (next_spare > 0) {
  84                 next_spare--;
  85                 if (fdlist[next_spare] >= 0)
  86                         close(fdlist[next_spare]);
  87         }
  88         free(fdlist);
  89         if (err) {
  90                 pr_err("Failed to restore critical section for reshape - sorry.\n");
  91                 if (!backup_file)
  92                         pr_err("Possibly you need to specify a --backup-file\n");
  93                 return 1;
  94         }
  95
  96         dprintf("restore_backup() returns status OK.\n");
  97         return 0;
  98 }
  99
 100 int Grow_Add_device(char *devname, int fd, char *newdev)
 101 {
 102         /* Add a device to an active array.
 103          * Currently, just extend a linear array.
 104          * This requires writing a new superblock on the
 105          * new device, calling the kernel to add the device,
 106          * and if that succeeds, update the superblock on
 107          * all other devices.
 108          * This means that we need to *find* all other devices.
 109          */
 110         struct mdinfo info;
 111
 112         struct stat stb;
 113         int nfd, fd2;
 114         int d, nd;
 115         struct supertype *st = NULL;
 116         char *subarray = NULL;
 117
 118         if (ioctl(fd, GET_ARRAY_INFO, &info.array) < 0) {
 119                 pr_err("cannot get array info for %s\n", devname);
 120                 return 1;
 121         }
 122
 123         if (info.array.level != -1) {
 124                 pr_err("can only add devices to linear arrays\n");
 125                 return 1;
 126         }
 127
 128         st = super_by_fd(fd, &subarray);
 129         if (!st) {
 130                 pr_err("cannot handle arrays with superblock version %d\n",
 131                        info.array.major_version);
 132                 return 1;
 133         }
 134
 135         if (subarray) {
 136                 pr_err("Cannot grow linear sub-arrays yet\n");
 137                 free(subarray);
 138                 free(st);
 139                 return 1;
 140         }
 141
 142         nfd = open(newdev, O_RDWR|O_EXCL|O_DIRECT);
 143         if (nfd < 0) {
 144                 pr_err("cannot open %s\n", newdev);
 145                 free(st);
 146                 return 1;
 147         }
 148         fstat(nfd, &stb);
 149         if ((stb.st_mode & S_IFMT) != S_IFBLK) {
 150                 pr_err("%s is not a block device!\n", newdev);
 151                 close(nfd);
 152                 free(st);
 153                 return 1;
 154         }
 155         /* now check out all the devices and make sure we can read the
 156          * superblock */
 157         for (d=0 ; d < info.array.raid_disks ; d++) {
 158                 mdu_disk_info_t disk;
 159                 char *dv;
 160
 161                 st->ss->free_super(st);
 162
 163                 disk.number = d;
 164                 if (ioctl(fd, GET_DISK_INFO, &disk) < 0) {
 165                         pr_err("cannot get device detail for device %d\n",
 166                                 d);
 167                         close(nfd);
 168                         free(st);
 169                         return 1;
 170                 }
 171                 dv = map_dev(disk.major, disk.minor, 1);
 172                 if (!dv) {
 173                         pr_err("cannot find device file for device %d\n",
 174                                 d);
 175                         close(nfd);
 176                         free(st);
 177                         return 1;
 178                 }
 179                 fd2 = dev_open(dv, O_RDWR);
 180                 if (fd2 < 0) {
 181                         pr_err("cannot open device file %s\n", dv);
 182                         close(nfd);
 183                         free(st);
 184                         return 1;
 185                 }
 186
 187                 if (st->ss->load_super(st, fd2, NULL)) {
 188                         pr_err("cannot find super block on %s\n", dv);
 189                         close(nfd);
 190                         close(fd2);
 191                         free(st);
 192                         return 1;
 193                 }
 194                 close(fd2);
 195         }
 196         /* Ok, looks good. Lets update the superblock and write it out to
 197          * newdev.
 198          */
 199
 200         info.disk.number = d;
 201         info.disk.major = major(stb.st_rdev);
 202         info.disk.minor = minor(stb.st_rdev);
 203         info.disk.raid_disk = d;
 204         info.disk.state = (1 << MD_DISK_SYNC) | (1 << MD_DISK_ACTIVE);
 205         st->ss->update_super(st, &info, "linear-grow-new", newdev,
 206                              0, 0, NULL);
 207
 208         if (st->ss->store_super(st, nfd)) {
 209                 pr_err("Cannot store new superblock on %s\n",
 210                         newdev);
 211                 close(nfd);
 212                 return 1;
 213         }
 214         close(nfd);
 215
 216         if (ioctl(fd, ADD_NEW_DISK, &info.disk) != 0) {
 217                 pr_err("Cannot add new disk to this array\n");
 218                 return 1;
 219         }
 220         /* Well, that seems to have worked.
 221          * Now go through and update all superblocks
 222          */
 223
 224         if (ioctl(fd, GET_ARRAY_INFO, &info.array) < 0) {
 225                 pr_err("cannot get array info for %s\n", devname);
 226                 return 1;
 227         }
 228
 229         nd = d;
 230         for (d=0 ; d < info.array.raid_disks ; d++) {
 231                 mdu_disk_info_t disk;
 232                 char *dv;
 233
 234                 disk.number = d;
 235                 if (ioctl(fd, GET_DISK_INFO, &disk) < 0) {
 236                         pr_err("cannot get device detail for device %d\n",
 237                                 d);
 238                         return 1;
 239                 }
 240                 dv = map_dev(disk.major, disk.minor, 1);
 241                 if (!dv) {
 242                         pr_err("cannot find device file for device %d\n",
 243                                 d);
 244                         return 1;
 245                 }
 246                 fd2 = dev_open(dv, O_RDWR);
 247                 if (fd2 < 0) {
 248                         pr_err("cannot open device file %s\n", dv);
 249                         return 1;
 250                 }
 251                 if (st->ss->load_super(st, fd2, NULL)) {
 252                         pr_err("cannot find super block on %s\n", dv);
 253                         close(fd);
 254                         return 1;
 255                 }
 256                 info.array.raid_disks = nd+1;
 257                 info.array.nr_disks = nd+1;
 258                 info.array.active_disks = nd+1;
 259                 info.array.working_disks = nd+1;
 260
 261                 st->ss->update_super(st, &info, "linear-grow-update", dv,
 262                                      0, 0, NULL);
 263
 264                 if (st->ss->store_super(st, fd2)) {
 265                         pr_err("Cannot store new superblock on %s\n", dv);
 266                         close(fd2);
 267                         return 1;
 268                 }
 269                 close(fd2);
 270         }
 271
 272         return 0;
 273 }
 274
 275 int Grow_addbitmap(char *devname, int fd, struct context *c, struct shape *s)
 276 {
 277         /*
 278          * First check that array doesn't have a bitmap
 279          * Then create the bitmap
 280          * Then add it
 281          *
 282          * For internal bitmaps, we need to check the version,
 283          * find all the active devices, and write the bitmap block
 284          * to all devices
 285          */
 286         mdu_bitmap_file_t bmf;
 287         mdu_array_info_t array;
 288         struct supertype *st;
 289         char *subarray = NULL;
 290         int major = BITMAP_MAJOR_HI;
 291         int vers = md_get_version(fd);
 292         unsigned long long bitmapsize, array_size;
 293
 294         if (vers < 9003) {
 295                 major = BITMAP_MAJOR_HOSTENDIAN;
 296                 pr_err("Warning - bitmaps created on this kernel are not portable\n"
 297                         "  between different architectures.  Consider upgrading the Linux kernel.\n");
 298         }
 299
 300         /*
 301          * We only ever get called if s->bitmap_file is != NULL, so this check
 302          * is just here to quiet down static code checkers.
 303          */
 304         if (!s->bitmap_file)
 305                 return 1;
 306
 307         if (strcmp(s->bitmap_file, "clustered") == 0)
 308                 major = BITMAP_MAJOR_CLUSTERED;
 309
 310         if (ioctl(fd, GET_BITMAP_FILE, &bmf) != 0) {
 311                 if (errno == ENOMEM)
 312                         pr_err("Memory allocation failure.\n");
 313                 else
 314                         pr_err("bitmaps not supported by this kernel.\n");
 315                 return 1;
 316         }
 317         if (bmf.pathname[0]) {
 318                 if (strcmp(s->bitmap_file,"none") == 0) {
 319                         if (ioctl(fd, SET_BITMAP_FILE, -1) != 0) {
 320                                 pr_err("failed to remove bitmap %s\n",
 321                                         bmf.pathname);
 322                                 return 1;
 323                         }
 324                         return 0;
 325                 }
 326                 pr_err("%s already has a bitmap (%s)\n",
 327                         devname, bmf.pathname);
 328                 return 1;
 329         }
 330         if (ioctl(fd, GET_ARRAY_INFO, &array) != 0) {
 331                 pr_err("cannot get array status for %s\n", devname);
 332                 return 1;
 333         }
 334         if (array.state & (1 << MD_SB_BITMAP_PRESENT)) {
 335                 if (strcmp(s->bitmap_file, "none")==0) {
 336                         array.state &= ~(1 << MD_SB_BITMAP_PRESENT);
 337                         if (ioctl(fd, SET_ARRAY_INFO, &array) != 0) {
 338                                 if (array.state & (1 << MD_SB_CLUSTERED))
 339                                         pr_err("failed to remove clustered bitmap.\n");
 340                                 else
 341                                         pr_err("failed to remove internal bitmap.\n");
 342                                 return 1;
 343                         }
 344                         return 0;
 345                 }
 346                 pr_err("bitmap already present on %s\n", devname);
 347                 return 1;
 348         }
 349
 350         if (strcmp(s->bitmap_file, "none") == 0) {
 351                 pr_err("no bitmap found on %s\n", devname);
 352                 return 1;
 353         }
 354         if (array.level <= 0) {
 355                 pr_err("Bitmaps not meaningful with level %s\n",
 356                         map_num(pers, array.level)?:"of this array");
 357                 return 1;
 358         }
 359         bitmapsize = array.size;
 360         bitmapsize <<= 1;
 361         if (get_dev_size(fd, NULL, &array_size) &&
 362             array_size > (0x7fffffffULL << 9)) {
 363                 /* Array is big enough that we cannot trust array.size
 364                  * try other approaches
 365                  */
 366                 bitmapsize = get_component_size(fd);
 367         }
 368         if (bitmapsize == 0) {
 369                 pr_err("Cannot reliably determine size of array to create bitmap - sorry.\n");
 370                 return 1;
 371         }
 372
 373         if (array.level == 10) {
 374                 int ncopies;
 375
 376                 ncopies = (array.layout & 255) * ((array.layout >> 8) & 255);
 377                 bitmapsize = bitmapsize * array.raid_disks / ncopies;
 378         }
 379
 380         st = super_by_fd(fd, &subarray);
 381         if (!st) {
 382                 pr_err("Cannot understand version %d.%d\n",
 383                         array.major_version, array.minor_version);
 384                 return 1;
 385         }
 386         if (subarray) {
 387                 pr_err("Cannot add bitmaps to sub-arrays yet\n");
 388                 free(subarray);
 389                 free(st);
 390                 return 1;
 391         }
 392         if (strcmp(s->bitmap_file, "internal") == 0 ||
 393             strcmp(s->bitmap_file, "clustered") == 0) {
 394                 int rv;
 395                 int d;
 396                 int offset_setable = 0;
 397                 struct mdinfo *mdi;
 398                 if (st->ss->add_internal_bitmap == NULL) {
 399                         pr_err("Internal bitmaps not supported with %s metadata\n", st->ss->name);
 400                         return 1;
 401                 }
 402                 st->nodes = c->nodes;
 403                 st->cluster_name = c->homecluster;
 404                 mdi = sysfs_read(fd, NULL, GET_BITMAP_LOCATION);
 405                 if (mdi)
 406                         offset_setable = 1;
 407                 for (d = 0; d < st->max_devs; d++) {
 408                         mdu_disk_info_t disk;
 409                         char *dv;
 410                         int fd2;
 411
 412                         disk.number = d;
 413                         if (ioctl(fd, GET_DISK_INFO, &disk) < 0)
 414                                 continue;
 415                         if (disk.major == 0 && disk.minor == 0)
 416                                 continue;
 417                         if ((disk.state & (1 << MD_DISK_SYNC)) == 0)
 418                                 continue;
 419                         dv = map_dev(disk.major, disk.minor, 1);
 420                         if (!dv)
 421                                 continue;
 422                         fd2 = dev_open(dv, O_RDWR);
 423                         if (fd2 < 0)
 424                                 continue;
 425                         rv = st->ss->load_super(st, fd2, NULL);
 426                         if (!rv) {
 427                                 rv = st->ss->add_internal_bitmap(
 428                                         st, &s->bitmap_chunk, c->delay,
 429                                         s->write_behind, bitmapsize,
 430                                         offset_setable, major);
 431                                 if (!rv) {
 432                                         st->ss->write_bitmap(st, fd2,
 433                                                              NodeNumUpdate);
 434                                 } else {
 435                                         pr_err("failed to create internal bitmap - chunksize problem.\n");
 436                                 }
 437                         } else {
 438                                 pr_err("failed to load super-block.\n");
 439                         }
 440                         close(fd2);
 441                         if (rv)
 442                                 return 1;
 443                 }
 444                 if (offset_setable) {
 445                         st->ss->getinfo_super(st, mdi, NULL);
 446                         sysfs_init(mdi, fd, NULL);
 447                         rv = sysfs_set_num_signed(mdi, NULL, "bitmap/location",
 448                                                   mdi->bitmap_offset);
 449                 } else {
 450                         if (strcmp(s->bitmap_file, "clustered") == 0)
 451                                 array.state |= (1 << MD_SB_CLUSTERED);
 452                         array.state |= (1 << MD_SB_BITMAP_PRESENT);
 453                         rv = ioctl(fd, SET_ARRAY_INFO, &array);
 454                 }
 455                 if (rv < 0) {
 456                         if (errno == EBUSY)
 457                                 pr_err("Cannot add bitmap while array is resyncing or reshaping etc.\n");
 458                         pr_err("failed to set internal bitmap.\n");
 459                         return 1;
 460                 }
 461         } else {
 462                 int uuid[4];
 463                 int bitmap_fd;
 464                 int d;
 465                 int max_devs = st->max_devs;
 466
 467                 /* try to load a superblock */
 468                 for (d = 0; d < max_devs; d++) {
 469                         mdu_disk_info_t disk;
 470                         char *dv;
 471                         int fd2;
 472                         disk.number = d;
 473                         if (ioctl(fd, GET_DISK_INFO, &disk) < 0)
 474                                 continue;
 475                         if ((disk.major==0 && disk.minor == 0) ||
 476                             (disk.state & (1 << MD_DISK_REMOVED)))
 477                                 continue;
 478                         dv = map_dev(disk.major, disk.minor, 1);
 479                         if (!dv)
 480                                 continue;
 481                         fd2 = dev_open(dv, O_RDONLY);
 482                         if (fd2 >= 0) {
 483                                 if (st->ss->load_super(st, fd2, NULL) == 0) {
 484                                         close(fd2);
 485                                         st->ss->uuid_from_super(st, uuid);
 486                                         break;
 487                                 }
 488                                 close(fd2);
 489                         }
 490                 }
 491                 if (d == max_devs) {
 492                         pr_err("cannot find UUID for array!\n");
 493                         return 1;
 494                 }
 495                 if (CreateBitmap(s->bitmap_file, c->force, (char*)uuid,
 496                                  s->bitmap_chunk, c->delay, s->write_behind,
 497                                  bitmapsize, major)) {
 498                         return 1;
 499                 }
 500                 bitmap_fd = open(s->bitmap_file, O_RDWR);
 501                 if (bitmap_fd < 0) {
 502                         pr_err("weird: %s cannot be opened\n", s->bitmap_file);
 503                         return 1;
 504                 }
 505                 if (ioctl(fd, SET_BITMAP_FILE, bitmap_fd) < 0) {
 506                         int err = errno;
 507                         if (errno == EBUSY)
 508                                 pr_err("Cannot add bitmap while array is resyncing or reshaping etc.\n");
 509                         pr_err("Cannot set bitmap file for %s: %s\n",
 510                                 devname, strerror(err));
 511                         return 1;
 512                 }
 513         }
 514
 515         return 0;
 516 }
 517
 518 /*
 519  * When reshaping an array we might need to backup some data.
 520  * This is written to all spares with a 'super_block' describing it.
 521  * The superblock goes 4K from the end of the used space on the
 522  * device.
 523  * It if written after the backup is complete.
 524  * It has the following structure.
 525  */
 526
 527 static struct mdp_backup_super {
 528         char    magic[16];  /* md_backup_data-1 or -2 */
 529         __u8    set_uuid[16];
 530         __u64   mtime;
 531         /* start/sizes in 512byte sectors */
 532         __u64   devstart;       /* address on backup device/file of data */
 533         __u64   arraystart;
 534         __u64   length;
 535         __u32   sb_csum;        /* csum of preceeding bytes. */
 536         __u32   pad1;
 537         __u64   devstart2;      /* offset in to data of second section */
 538         __u64   arraystart2;
 539         __u64   length2;
 540         __u32   sb_csum2;       /* csum of preceeding bytes. */
 541         __u8 pad[512-68-32];
 542 } __attribute__((aligned(512))) bsb, bsb2;
 543
 544 static __u32 bsb_csum(char *buf, int len)
 545 {
 546         int i;
 547         int csum = 0;
 548         for (i = 0; i < len; i++)
 549                 csum = (csum<<3) + buf[0];
 550         return __cpu_to_le32(csum);
 551 }
 552
 553 static int check_idle(struct supertype *st)
 554 {
 555         /* Check that all member arrays for this container, or the
 556          * container of this array, are idle
 557          */
 558         char *container = (st->container_devnm[0]
 559                            ? st->container_devnm : st->devnm);
 560         struct mdstat_ent *ent, *e;
 561         int is_idle = 1;
 562
 563         ent = mdstat_read(0, 0);
 564         for (e = ent ; e; e = e->next) {
 565                 if (!is_container_member(e, container))
 566                         continue;
 567                 if (e->percent >= 0) {
 568                         is_idle = 0;
 569                         break;
 570                 }
 571         }
 572         free_mdstat(ent);
 573         return is_idle;
 574 }
 575
 576 static int freeze_container(struct supertype *st)
 577 {
 578         char *container = (st->container_devnm[0]
 579                            ? st->container_devnm : st->devnm);
 580
 581         if (!check_idle(st))
 582                 return -1;
 583
 584         if (block_monitor(container, 1)) {
 585                 pr_err("failed to freeze container\n");
 586                 return -2;
 587         }
 588
 589         return 1;
 590 }
 591
 592 static void unfreeze_container(struct supertype *st)
 593 {
 594         char *container = (st->container_devnm[0]
 595                            ? st->container_devnm : st->devnm);
 596
 597         unblock_monitor(container, 1);
 598 }
 599
 600 static int freeze(struct supertype *st)
 601 {
 602         /* Try to freeze resync/rebuild on this array/container.
 603          * Return -1 if the array is busy,
 604          * return -2 container cannot be frozen,
 605          * return 0 if this kernel doesn't support 'frozen'
 606          * return 1 if it worked.
 607          */
 608         if (st->ss->external)
 609                 return freeze_container(st);
 610         else {
 611                 struct mdinfo *sra = sysfs_read(-1, st->devnm, GET_VERSION);
 612                 int err;
 613                 char buf[20];
 614
 615                 if (!sra)
 616                         return -1;
 617                 /* Need to clear any 'read-auto' status */
 618                 if (sysfs_get_str(sra, NULL, "array_state", buf, 20) > 0 &&
 619                     strncmp(buf, "read-auto", 9) == 0)
 620                         sysfs_set_str(sra, NULL, "array_state", "clean");
 621
 622                 err = sysfs_freeze_array(sra);
 623                 sysfs_free(sra);
 624                 return err;
 625         }
 626 }
 627
 628 static void unfreeze(struct supertype *st)
 629 {
 630         if (st->ss->external)
 631                 return unfreeze_container(st);
 632         else {
 633                 struct mdinfo *sra = sysfs_read(-1, st->devnm, GET_VERSION);
 634                 char buf[20];
 635
 636                 if (sra &&
 637                     sysfs_get_str(sra, NULL, "sync_action", buf, 20) > 0
 638                     && strcmp(buf, "frozen\n") == 0)
 639                         sysfs_set_str(sra, NULL, "sync_action", "idle");
 640                 sysfs_free(sra);
 641         }
 642 }
 643
 644 static void wait_reshape(struct mdinfo *sra)
 645 {
 646         int fd = sysfs_get_fd(sra, NULL, "sync_action");
 647         char action[20];
 648
 649         if (fd < 0)
 650                 return;
 651
 652         while (sysfs_fd_get_str(fd, action, 20) > 0 &&
 653                strncmp(action, "reshape", 7) == 0)
 654                 sysfs_wait(fd, NULL);
 655         close(fd);
 656 }
 657
 658 static int reshape_super(struct supertype *st, unsigned long long size,
 659                          int level, int layout, int chunksize, int raid_disks,
 660                          int delta_disks, char *backup_file, char *dev,
 661                          int direction, int verbose)
 662 {
 663         /* nothing extra to check in the native case */
 664         if (!st->ss->external)
 665                 return 0;
 666         if (!st->ss->reshape_super ||
 667             !st->ss->manage_reshape) {
 668                 pr_err("%s metadata does not support reshape\n",
 669                         st->ss->name);
 670                 return 1;
 671         }
 672
 673         return st->ss->reshape_super(st, size, level, layout, chunksize,
 674                                      raid_disks, delta_disks, backup_file, dev,
 675                                      direction, verbose);
 676 }
 677
 678 static void sync_metadata(struct supertype *st)
 679 {
 680         if (st->ss->external) {
 681                 if (st->update_tail) {
 682                         flush_metadata_updates(st);
 683                         st->update_tail = &st->updates;
 684                 } else
 685                         st->ss->sync_metadata(st);
 686         }
 687 }
 688
 689 static int subarray_set_num(char *container, struct mdinfo *sra, char *name, int n)
 690 {
 691         /* when dealing with external metadata subarrays we need to be
 692          * prepared to handle EAGAIN.  The kernel may need to wait for
 693          * mdmon to mark the array active so the kernel can handle
 694          * allocations/writeback when preparing the reshape action
 695          * (md_allow_write()).  We temporarily disable safe_mode_delay
 696          * to close a race with the array_state going clean before the
 697          * next write to raid_disks / stripe_cache_size
 698          */
 699         char safe[50];
 700         int rc;
 701
 702         /* only 'raid_disks' and 'stripe_cache_size' trigger md_allow_write */
 703         if (!container ||
 704             (strcmp(name, "raid_disks") != 0 &&
 705              strcmp(name, "stripe_cache_size") != 0))
 706                 return sysfs_set_num(sra, NULL, name, n);
 707
 708         rc = sysfs_get_str(sra, NULL, "safe_mode_delay", safe, sizeof(safe));
 709         if (rc <= 0)
 710                 return -1;
 711         sysfs_set_num(sra, NULL, "safe_mode_delay", 0);
 712         rc = sysfs_set_num(sra, NULL, name, n);
 713         if (rc < 0 && errno == EAGAIN) {
 714                 ping_monitor(container);
 715                 /* if we get EAGAIN here then the monitor is not active
 716                  * so stop trying
 717                  */
 718                 rc = sysfs_set_num(sra, NULL, name, n);
 719         }
 720         sysfs_set_str(sra, NULL, "safe_mode_delay", safe);
 721         return rc;
 722 }
 723
 724 int start_reshape(struct mdinfo *sra, int already_running,
 725                   int before_data_disks, int data_disks)
 726 {
 727         int err;
 728         unsigned long long sync_max_to_set;
 729
 730         sysfs_set_num(sra, NULL, "suspend_lo", 0x7FFFFFFFFFFFFFFFULL);
 731         err = sysfs_set_num(sra, NULL, "suspend_hi", sra->reshape_progress);
 732         err = err ?: sysfs_set_num(sra, NULL, "suspend_lo",
 733                                    sra->reshape_progress);
 734         if (before_data_disks <= data_disks)
 735                 sync_max_to_set = sra->reshape_progress / data_disks;
 736         else
 737                 sync_max_to_set = (sra->component_size * data_disks
 738                                    - sra->reshape_progress) / data_disks;
 739         if (!already_running)
 740                 sysfs_set_num(sra, NULL, "sync_min", sync_max_to_set);
 741         err = err ?: sysfs_set_num(sra, NULL, "sync_max", sync_max_to_set);
 742         if (!already_running && err == 0) {
 743                 int cnt = 5;
 744                 do {
 745                         err = sysfs_set_str(sra, NULL, "sync_action", "reshape");
 746                         if (err)
 747                                 sleep(1);
 748                 } while (err && errno == EBUSY && cnt-- > 0);
 749         }
 750         return err;
 751 }
 752
 753 void abort_reshape(struct mdinfo *sra)
 754 {
 755         sysfs_set_str(sra, NULL, "sync_action", "idle");
 756         /*
 757          * Prior to kernel commit: 23ddff3792f6 ("md: allow suspend_lo and
 758          * suspend_hi to decrease as well as increase.")
 759          * you could only increase suspend_{lo,hi} unless the region they
 760          * covered was empty.  So to reset to 0, you need to push suspend_lo
 761          * up past suspend_hi first.  So to maximize the chance of mdadm
 762          * working on all kernels, we want to keep doing that.
 763          */
 764         sysfs_set_num(sra, NULL, "suspend_lo", 0x7FFFFFFFFFFFFFFFULL);
 765         sysfs_set_num(sra, NULL, "suspend_hi", 0);
 766         sysfs_set_num(sra, NULL, "suspend_lo", 0);
 767         sysfs_set_num(sra, NULL, "sync_min", 0);
 768         // It isn't safe to reset sync_max as we aren't monitoring.
 769         // Array really should be stopped at this point.
 770 }
 771
 772 int remove_disks_for_takeover(struct supertype *st,
 773                               struct mdinfo *sra,
 774                               int layout)
 775 {
 776         int nr_of_copies;
 777         struct mdinfo *remaining;
 778         int slot;
 779
 780         if (st->ss->external) {
 781                 int rv = 0;
 782                 struct mdinfo *arrays = st->ss->container_content(st, NULL);
 783                 /*
 784                  * containter_content returns list of arrays in container
 785                  * If arrays->next is not NULL it means that there are
 786                  * 2 arrays in container and operation should be blocked
 787                  */
 788                 if (arrays) {
 789                         if (arrays->next)
 790                                 rv = 1;
 791                         sysfs_free(arrays);
 792                         if (rv) {
 793                                 pr_err("Error. Cannot perform operation on /dev/%s\n", st->devnm);
 794                                 pr_err("For this operation it MUST be single array in container\n");
 795                                 return rv;
 796                         }
 797                 }
 798         }
 799
 800         if (sra->array.level == 10)
 801                 nr_of_copies = layout & 0xff;
 802         else if (sra->array.level == 1)
 803                 nr_of_copies = sra->array.raid_disks;
 804         else
 805                 return 1;
 806
 807         remaining = sra->devs;
 808         sra->devs = NULL;
 809         /* for each 'copy', select one device and remove from the list. */
 810         for (slot = 0; slot < sra->array.raid_disks; slot += nr_of_copies) {
 811                 struct mdinfo **diskp;
 812                 int found = 0;
 813
 814                 /* Find a working device to keep */
 815                 for (diskp =  &remaining; *diskp ; diskp = &(*diskp)->next) {
 816                         struct mdinfo *disk = *diskp;
 817
 818                         if (disk->disk.raid_disk < slot)
 819                                 continue;
 820                         if (disk->disk.raid_disk >= slot + nr_of_copies)
 821                                 continue;
 822                         if (disk->disk.state & (1<<MD_DISK_REMOVED))
 823                                 continue;
 824                         if (disk->disk.state & (1<<MD_DISK_FAULTY))
 825                                 continue;
 826                         if (!(disk->disk.state & (1<<MD_DISK_SYNC)))
 827                                 continue;
 828
 829                         /* We have found a good disk to use! */
 830                         *diskp = disk->next;
 831                         disk->next = sra->devs;
 832                         sra->devs = disk;
 833                         found = 1;
 834                         break;
 835                 }
 836                 if (!found)
 837                         break;
 838         }
 839
 840         if (slot < sra->array.raid_disks) {
 841                 /* didn't find all slots */
 842                 struct mdinfo **e;
 843                 e = &remaining;
 844                 while (*e)
 845                         e = &(*e)->next;
 846                 *e = sra->devs;
 847                 sra->devs = remaining;
 848                 return 1;
 849         }
 850
 851         /* Remove all 'remaining' devices from the array */
 852         while (remaining) {
 853                 struct mdinfo *sd = remaining;
 854                 remaining = sd->next;
 855
 856                 sysfs_set_str(sra, sd, "state", "faulty");
 857                 sysfs_set_str(sra, sd, "slot", "none");
 858                 /* for external metadata disks should be removed in mdmon */
 859                 if (!st->ss->external)
 860                         sysfs_set_str(sra, sd, "state", "remove");
 861                 sd->disk.state |= (1<<MD_DISK_REMOVED);
 862                 sd->disk.state &= ~(1<<MD_DISK_SYNC);
 863                 sd->next = sra->devs;
 864                 sra->devs = sd;
 865         }
 866         return 0;
 867 }
 868
 869 void reshape_free_fdlist(int *fdlist,
 870                          unsigned long long *offsets,
 871                          int size)
 872 {
 873         int i;
 874
 875         for (i = 0; i < size; i++)
 876                 if (fdlist[i] >= 0)
 877                         close(fdlist[i]);
 878
 879         free(fdlist);
 880         free(offsets);
 881 }
 882
 883 int reshape_prepare_fdlist(char *devname,
 884                            struct mdinfo *sra,
 885                            int raid_disks,
 886                            int nrdisks,
 887                            unsigned long blocks,
 888                            char *backup_file,
 889                            int *fdlist,
 890                            unsigned long long *offsets)
 891 {
 892         int d = 0;
 893         struct mdinfo *sd;
 894
 895         enable_fds(nrdisks);
 896         for (d = 0; d <= nrdisks; d++)
 897                 fdlist[d] = -1;
 898         d = raid_disks;
 899         for (sd = sra->devs; sd; sd = sd->next) {
 900                 if (sd->disk.state & (1<<MD_DISK_FAULTY))
 901                         continue;
 902                 if (sd->disk.state & (1<<MD_DISK_SYNC) &&
 903                     sd->disk.raid_disk < raid_disks) {
 904                         char *dn = map_dev(sd->disk.major,
 905                                            sd->disk.minor, 1);
 906                         fdlist[sd->disk.raid_disk]
 907                                 = dev_open(dn, O_RDONLY);
 908                         offsets[sd->disk.raid_disk] = sd->data_offset*512;
 909                         if (fdlist[sd->disk.raid_disk] < 0) {
 910                                 pr_err("%s: cannot open component %s\n",
 911                                        devname, dn ? dn : "-unknown-");
 912                                 d = -1;
 913                                 goto release;
 914                         }
 915                 } else if (backup_file == NULL) {
 916                         /* spare */
 917                         char *dn = map_dev(sd->disk.major,
 918                                            sd->disk.minor, 1);
 919                                 fdlist[d] = dev_open(dn, O_RDWR);
 920                                 offsets[d] = (sd->data_offset + sra->component_size - blocks - 8)*512;
 921                                 if (fdlist[d] < 0) {
 922                                         pr_err("%s: cannot open component %s\n",
 923                                                 devname, dn ? dn : "-unknown-");
 924                                         d = -1;
 925                                         goto release;
 926                                 }
 927                                 d++;
 928                         }
 929                 }
 930 release:
 931         return d;
 932 }
 933
 934 int reshape_open_backup_file(char *backup_file,
 935                              int fd,
 936                              char *devname,
 937                              long blocks,
 938                              int *fdlist,
 939                              unsigned long long *offsets,
 940                              char *sys_name,
 941                              int restart)
 942 {
 943         /* Return 1 on success, 0 on any form of failure */
 944         /* need to check backup file is large enough */
 945         char buf[512];
 946         struct stat stb;
 947         unsigned int dev;
 948         int i;
 949
 950         *fdlist = open(backup_file, O_RDWR|O_CREAT|(restart ? O_TRUNC : O_EXCL),
 951                        S_IRUSR | S_IWUSR);
 952         *offsets = 8 * 512;
 953         if (*fdlist < 0) {
 954                 pr_err("%s: cannot create backup file %s: %s\n",
 955                         devname, backup_file, strerror(errno));
 956                 return 0;
 957         }
 958         /* Guard against backup file being on array device.
 959          * If array is partitioned or if LVM etc is in the
 960          * way this will not notice, but it is better than
 961          * nothing.
 962          */
 963         fstat(*fdlist, &stb);
 964         dev = stb.st_dev;
 965         fstat(fd, &stb);
 966         if (stb.st_rdev == dev) {
 967                 pr_err("backup file must NOT be on the array being reshaped.\n");
 968                 close(*fdlist);
 969                 return 0;
 970         }
 971
 972         memset(buf, 0, 512);
 973         for (i=0; i < blocks + 8 ; i++) {
 974                 if (write(*fdlist, buf, 512) != 512) {
 975                         pr_err("%s: cannot create backup file %s: %s\n",
 976                                 devname, backup_file, strerror(errno));
 977                         return 0;
 978                 }
 979         }
 980         if (fsync(*fdlist) != 0) {
 981                 pr_err("%s: cannot create backup file %s: %s\n",
 982                         devname, backup_file, strerror(errno));
 983                 return 0;
 984         }
 985
 986         if (!restart && strncmp(backup_file, MAP_DIR, strlen(MAP_DIR)) != 0) {
 987                 char *bu = make_backup(sys_name);
 988                 if (symlink(backup_file, bu))
 989                         pr_err("Recording backup file in " MAP_DIR " failed: %s\n",
 990                                strerror(errno));
 991                 free(bu);
 992         }
 993
 994         return 1;
 995 }
 996
 997 unsigned long compute_backup_blocks(int nchunk, int ochunk,
 998                                     unsigned int ndata, unsigned int odata)
 999 {
1000         unsigned long a, b, blocks;
1001         /* So how much do we need to backup.
1002          * We need an amount of data which is both a whole number of
1003          * old stripes and a whole number of new stripes.
1004          * So LCM for (chunksize*datadisks).
1005          */
1006         a = (ochunk/512) * odata;
1007         b = (nchunk/512) * ndata;
1008         /* Find GCD */
1009         a = GCD(a, b);
1010         /* LCM == product / GCD */
1011         blocks = (ochunk/512) * (nchunk/512) * odata * ndata / a;
1012
1013         return blocks;
1014 }
1015
1016 char *analyse_change(char *devname, struct mdinfo *info, struct reshape *re)
1017 {
1018         /* Based on the current array state in info->array and
1019          * the changes in info->new_* etc, determine:
1020          *  - whether the change is possible
1021          *  - Intermediate level/raid_disks/layout
1022          *  - whether a restriping reshape is needed
1023          *  - number of sectors in minimum change unit.  This
1024          *    will cover a whole number of stripes in 'before' and
1025          *    'after'.
1026          *
1027          * Return message if the change should be rejected
1028          *        NULL if the change can be achieved
1029          *
1030          * This can be called as part of starting a reshape, or
1031          * when assembling an array that is undergoing reshape.
1032          */
1033         int near, far, offset, copies;
1034         int new_disks;
1035         int old_chunk, new_chunk;
1036         /* delta_parity records change in number of devices
1037          * caused by level change
1038          */
1039         int delta_parity = 0;
1040
1041         memset(re, 0, sizeof(*re));
1042
1043         /* If a new level not explicitly given, we assume no-change */
1044         if (info->new_level == UnSet)
1045                 info->new_level = info->array.level;
1046
1047         if (info->new_chunk)
1048                 switch (info->new_level) {
1049                 case 0:
1050                 case 4:
1051                 case 5:
1052                 case 6:
1053                 case 10:
1054                         /* chunk size is meaningful, must divide component_size
1055                          * evenly
1056                          */
1057                         if (info->component_size % (info->new_chunk/512)) {
1058                                 unsigned long long shrink = info->component_size;
1059                                 shrink &= ~(unsigned long long)(info->new_chunk/512-1);
1060                                 pr_err("New chunk size (%dK) does not evenly divide device size (%lluk)\n",
1061                                        info->new_chunk/1024, info->component_size/2);
1062                                 pr_err("After shrinking any filesystem, \"mdadm --grow %s --size %llu\"\n",
1063                                        devname, shrink/2);
1064                                 pr_err("will shrink the array so the given chunk size would work.\n");
1065                                 return "";
1066                         }
1067                         break;
1068                 default:
1069                         return "chunk size not meaningful for this level";
1070                 }
1071         else
1072                 info->new_chunk = info->array.chunk_size;
1073
1074         switch (info->array.level) {
1075         default:
1076                 return "No reshape is possibly for this RAID level";
1077         case LEVEL_LINEAR:
1078                 if (info->delta_disks != UnSet)
1079                         return "Only --add is supported for LINEAR, setting --raid-disks is not needed";
1080                 else
1081                         return "Only --add is supported for LINEAR, other --grow options are not meaningful";
1082         case 1:
1083                 /* RAID1 can convert to RAID1 with different disks, or
1084                  * raid5 with 2 disks, or
1085                  * raid0 with 1 disk
1086                  */
1087                 if (info->new_level > 1 &&
1088                     (info->component_size & 7))
1089                         return "Cannot convert RAID1 of this size - reduce size to multiple of 4K first.";
1090                 if (info->new_level == 0) {
1091                         if (info->delta_disks != UnSet &&
1092                             info->delta_disks != 0)
1093                                 return "Cannot change number of disks with RAID1->RAID0 conversion";
1094                         re->level = 0;
1095                         re->before.data_disks = 1;
1096                         re->after.data_disks = 1;
1097                         return NULL;
1098                 }
1099                 if (info->new_level == 1) {
1100                         if (info->delta_disks == UnSet)
1101                                 /* Don't know what to do */
1102                                 return "no change requested for Growing RAID1";
1103                         re->level = 1;
1104                         return NULL;
1105                 }
1106                 if (info->array.raid_disks != 2 &&
1107                     info->new_level == 5)
1108                         return "Can only convert a 2-device array to RAID5";
1109                 if (info->array.raid_disks == 2 &&
1110                     info->new_level == 5) {
1111
1112                         re->level = 5;
1113                         re->before.data_disks = 1;
1114                         if (info->delta_disks != UnSet &&
1115                             info->delta_disks != 0)
1116                                 re->after.data_disks = 1 + info->delta_disks;
1117                         else
1118                                 re->after.data_disks = 1;
1119                         if (re->after.data_disks < 1)
1120                                 return "Number of disks too small for RAID5";
1121
1122                         re->before.layout = ALGORITHM_LEFT_SYMMETRIC;
1123                         info->array.chunk_size = 65536;
1124                         break;
1125                 }
1126                 /* Could do some multi-stage conversions, but leave that to
1127                  * later.
1128                  */
1129                 return "Impossibly level change request for RAID1";
1130
1131         case 10:
1132                 /* RAID10 can be converted from near mode to
1133                  * RAID0 by removing some devices.
1134                  * It can also be reshaped if the kernel supports
1135                  * new_data_offset.
1136                  */
1137                 switch (info->new_level) {
1138                 case 0:
1139                         if ((info->array.layout & ~0xff) != 0x100)
1140                                 return "Cannot Grow RAID10 with far/offset layout";
1141                         /* number of devices must be multiple of number of copies */
1142                         if (info->array.raid_disks % (info->array.layout & 0xff))
1143                                 return "RAID10 layout too complex for Grow operation";
1144
1145                         new_disks = (info->array.raid_disks
1146                                      / (info->array.layout & 0xff));
1147                         if (info->delta_disks == UnSet)
1148                                 info->delta_disks = (new_disks
1149                                                      - info->array.raid_disks);
1150
1151                         if (info->delta_disks != new_disks - info->array.raid_disks)
1152                                 return "New number of raid-devices impossible for RAID10";
1153                         if (info->new_chunk &&
1154                             info->new_chunk != info->array.chunk_size)
1155                                 return "Cannot change chunk-size with RAID10 Grow";
1156
1157                         /* looks good */
1158                         re->level = 0;
1159                         re->before.data_disks = new_disks;
1160                         re->after.data_disks = re->before.data_disks;
1161                         return NULL;
1162
1163                 case 10:
1164                         near = info->array.layout & 0xff;
1165                         far = (info->array.layout >> 8) & 0xff;
1166                         offset = info->array.layout & 0x10000;
1167                         if (far > 1 && !offset)
1168                                 return "Cannot reshape RAID10 in far-mode";
1169                         copies = near * far;
1170
1171                         old_chunk = info->array.chunk_size * far;
1172
1173                         if (info->new_layout == UnSet)
1174                                 info->new_layout = info->array.layout;
1175                         else {
1176                                 near = info->new_layout & 0xff;
1177                                 far = (info->new_layout >> 8) & 0xff;
1178                                 offset = info->new_layout & 0x10000;
1179                                 if (far > 1 && !offset)
1180                                         return "Cannot reshape RAID10 to far-mode";
1181                                 if (near * far != copies)
1182                                         return "Cannot change number of copies when reshaping RAID10";
1183                         }
1184                         if (info->delta_disks == UnSet)
1185                                 info->delta_disks = 0;
1186                         new_disks = (info->array.raid_disks +
1187                                      info->delta_disks);
1188
1189                         new_chunk = info->new_chunk * far;
1190
1191                         re->level = 10;
1192                         re->before.layout = info->array.layout;
1193                         re->before.data_disks = info->array.raid_disks;
1194                         re->after.layout = info->new_layout;
1195                         re->after.data_disks = new_disks;
1196                         /* For RAID10 we don't do backup but do allow reshape,
1197                          * so set backup_blocks to INVALID_SECTORS rather than
1198                          * zero.
1199                          * And there is no need to synchronise stripes on both
1200                          * 'old' and  'new'.  So the important
1201                          * number is the minimum data_offset difference
1202                          * which is the larger of (offset copies * chunk).
1203                          */
1204                         re->backup_blocks = INVALID_SECTORS;
1205                         re->min_offset_change = max(old_chunk, new_chunk) / 512;
1206                         if (new_disks < re->before.data_disks &&
1207                             info->space_after < re->min_offset_change)
1208                                 /* Reduce component size by one chunk */
1209                                 re->new_size = (info->component_size -
1210                                                 re->min_offset_change);
1211                         else
1212                                 re->new_size = info->component_size;
1213                         re->new_size = re->new_size * new_disks / copies;
1214                         return NULL;
1215
1216                 default:
1217                         return "RAID10 can only be changed to RAID0";
1218                 }
1219         case 0:
1220                 /* RAID0 can be converted to RAID10, or to RAID456 */
1221                 if (info->new_level == 10) {
1222                         if (info->new_layout == UnSet && info->delta_disks == UnSet) {
1223                                 /* Assume near=2 layout */
1224                                 info->new_layout = 0x102;
1225                                 info->delta_disks = info->array.raid_disks;
1226                         }
1227                         if (info->new_layout == UnSet) {
1228                                 int copies = 1 + (info->delta_disks
1229                                                   / info->array.raid_disks);
1230                                 if (info->array.raid_disks * (copies-1)
1231                                     != info->delta_disks)
1232                                         return "Impossible number of devices for RAID0->RAID10";
1233                                 info->new_layout = 0x100 + copies;
1234                         }
1235                         if (info->delta_disks == UnSet) {
1236                                 int copies = info->new_layout & 0xff;
1237                                 if (info->new_layout != 0x100 + copies)
1238                                         return "New layout impossible for RAID0->RAID10";;
1239                                 info->delta_disks = (copies - 1) *
1240                                         info->array.raid_disks;
1241                         }
1242                         if (info->new_chunk &&
1243                             info->new_chunk != info->array.chunk_size)
1244                                 return "Cannot change chunk-size with RAID0->RAID10";
1245                         /* looks good */
1246                         re->level = 10;
1247                         re->before.data_disks = (info->array.raid_disks +
1248                                                  info->delta_disks);
1249                         re->after.data_disks = re->before.data_disks;
1250                         re->before.layout = info->new_layout;
1251                         return NULL;
1252                 }
1253
1254                 /* RAID0 can also covert to RAID0/4/5/6 by first converting to
1255                  * a raid4 style layout of the final level.
1256                  */
1257                 switch (info->new_level) {
1258                 case 4:
1259                         delta_parity = 1;
1260                 case 0:
1261                         re->level = 4;
1262                         re->before.layout = 0;
1263                         break;
1264                 case 5:
1265                         delta_parity = 1;
1266                         re->level = 5;
1267                         re->before.layout = ALGORITHM_PARITY_N;
1268                         if (info->new_layout == UnSet)
1269                                 info->new_layout = map_name(r5layout, "default");
1270                         break;
1271                 case 6:
1272                         delta_parity = 2;
1273                         re->level = 6;
1274                         re->before.layout = ALGORITHM_PARITY_N;
1275                         if (info->new_layout == UnSet)
1276                                 info->new_layout = map_name(r6layout, "default");
1277                         break;
1278                 default:
1279                         return "Impossible level change requested";
1280                 }
1281                 re->before.data_disks = info->array.raid_disks;
1282                 /* determining 'after' layout happens outside this 'switch' */
1283                 break;
1284
1285         case 4:
1286                 info->array.layout = ALGORITHM_PARITY_N;
1287         case 5:
1288                 switch (info->new_level) {
1289                 case 0:
1290                         delta_parity = -1;
1291                 case 4:
1292                         re->level = info->array.level;
1293                         re->before.data_disks = info->array.raid_disks - 1;
1294                         re->before.layout = info->array.layout;
1295                         break;
1296                 case 5:
1297                         re->level = 5;
1298                         re->before.data_disks = info->array.raid_disks - 1;
1299                         re->before.layout = info->array.layout;
1300                         break;
1301                 case 6:
1302                         delta_parity = 1;
1303                         re->level = 6;
1304                         re->before.data_disks = info->array.raid_disks - 1;
1305                         switch (info->array.layout) {
1306                         case ALGORITHM_LEFT_ASYMMETRIC:
1307                                 re->before.layout = ALGORITHM_LEFT_ASYMMETRIC_6;
1308                                 break;
1309                         case ALGORITHM_RIGHT_ASYMMETRIC:
1310                                 re->before.layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
1311                                 break;
1312                         case ALGORITHM_LEFT_SYMMETRIC:
1313                                 re->before.layout = ALGORITHM_LEFT_SYMMETRIC_6;
1314                                 break;
1315                         case ALGORITHM_RIGHT_SYMMETRIC:
1316                                 re->before.layout = ALGORITHM_RIGHT_SYMMETRIC_6;
1317                                 break;
1318                         case ALGORITHM_PARITY_0:
1319                                 re->before.layout = ALGORITHM_PARITY_0_6;
1320                                 break;
1321                         case ALGORITHM_PARITY_N:
1322                                 re->before.layout = ALGORITHM_PARITY_N_6;
1323                                 break;
1324                         default:
1325                                 return "Cannot convert an array with this layout";
1326                         }
1327                         break;
1328                 case 1:
1329                         if (info->array.raid_disks != 2)
1330                                 return "Can only convert a 2-device array to RAID1";
1331                         if (info->delta_disks != UnSet &&
1332                             info->delta_disks != 0)
1333                                 return "Cannot set raid_disk when converting RAID5->RAID1";
1334                         re->level = 1;
1335                         info->new_chunk = 0;
1336                         return NULL;
1337                 default:
1338                         return "Impossible level change requested";
1339                 }
1340                 break;
1341         case 6:
1342                 switch (info->new_level) {
1343                 case 4:
1344                 case 5:
1345                         delta_parity = -1;
1346                 case 6:
1347                         re->level = 6;
1348                         re->before.data_disks = info->array.raid_disks - 2;
1349                         re->before.layout = info->array.layout;
1350                         break;
1351                 default:
1352                         return "Impossible level change requested";
1353                 }
1354                 break;
1355         }
1356
1357         /* If we reached here then it looks like a re-stripe is
1358          * happening.  We have determined the intermediate level
1359          * and initial raid_disks/layout and stored these in 're'.
1360          *
1361          * We need to deduce the final layout that can be atomically
1362          * converted to the end state.
1363          */
1364         switch (info->new_level) {
1365         case 0:
1366                 /* We can only get to RAID0 from RAID4 or RAID5
1367                  * with appropriate layout and one extra device
1368                  */
1369                 if (re->level != 4 && re->level != 5)
1370                         return "Cannot covert to RAID0 from this level";
1371
1372                 switch (re->level) {
1373                 case 4:
1374                         re->before.layout = 0;
1375                         re->after.layout = 0;
1376                         break;
1377                 case 5:
1378                         re->after.layout = ALGORITHM_PARITY_N;
1379                         break;
1380                 }
1381                 break;
1382
1383         case 4:
1384                 /* We can only get to RAID4 from RAID5 */
1385                 if (re->level != 4 && re->level != 5)
1386                         return "Cannot convert to RAID4 from this level";
1387
1388                 switch (re->level) {
1389                 case 4:
1390                         re->after.layout = 0;
1391                         break;
1392                 case 5:
1393                         re->after.layout = ALGORITHM_PARITY_N;
1394                         break;
1395                 }
1396                 break;
1397
1398         case 5:
1399                 /* We get to RAID5 from RAID5 or RAID6 */
1400                 if (re->level != 5 && re->level != 6)
1401                         return "Cannot convert to RAID5 from this level";
1402
1403                 switch (re->level) {
1404                 case 5:
1405                         if (info->new_layout == UnSet)
1406                                 re->after.layout = re->before.layout;
1407                         else
1408                                 re->after.layout = info->new_layout;
1409                         break;
1410                 case 6:
1411                         if (info->new_layout == UnSet)
1412                                 info->new_layout = re->before.layout;
1413
1414                         /* after.layout needs to be raid6 version of new_layout */
1415                         if (info->new_layout == ALGORITHM_PARITY_N)
1416                                 re->after.layout = ALGORITHM_PARITY_N;
1417                         else {
1418                                 char layout[40];
1419                                 char *ls = map_num(r5layout, info->new_layout);
1420                                 int l;
1421                                 if (ls) {
1422                                         /* Current RAID6 layout has a RAID5
1423                                          * equivalent - good
1424                                          */
1425                                         strcat(strcpy(layout, ls), "-6");
1426                                         l = map_name(r6layout, layout);
1427                                         if (l == UnSet)
1428                                                 return "Cannot find RAID6 layout to convert to";
1429                                 } else {
1430                                         /* Current RAID6 has no equivalent.
1431                                          * If it is already a '-6' layout we
1432                                          * can leave it unchanged, else we must
1433                                          * fail
1434                                          */
1435                                         ls = map_num(r6layout, info->new_layout);
1436                                         if (!ls ||
1437                                             strcmp(ls+strlen(ls)-2, "-6") != 0)
1438                                                 return "Please specify new layout";
1439                                         l = info->new_layout;
1440                                 }
1441                                 re->after.layout = l;
1442                         }
1443                 }
1444                 break;
1445
1446         case 6:
1447                 /* We must already be at level 6 */
1448                 if (re->level != 6)
1449                         return "Impossible level change";
1450                 if (info->new_layout == UnSet)
1451                         re->after.layout = info->array.layout;
1452                 else
1453                         re->after.layout = info->new_layout;
1454                 break;
1455         default:
1456                 return "Impossible level change requested";
1457         }
1458         if (info->delta_disks == UnSet)
1459                 info->delta_disks = delta_parity;
1460
1461         re->after.data_disks = (re->before.data_disks
1462                                 + info->delta_disks
1463                                 - delta_parity);
1464         switch (re->level) {
1465         case 6: re->parity = 2;
1466                 break;
1467         case 4:
1468         case 5: re->parity = 1;
1469                 break;
1470         default: re->parity = 0;
1471                 break;
1472         }
1473         /* So we have a restripe operation, we need to calculate the number
1474          * of blocks per reshape operation.
1475          */
1476         re->new_size = info->component_size * re->before.data_disks;
1477         if (info->new_chunk == 0)
1478                 info->new_chunk = info->array.chunk_size;
1479         if (re->after.data_disks == re->before.data_disks &&
1480             re->after.layout == re->before.layout &&
1481             info->new_chunk == info->array.chunk_size) {
1482                 /* Nothing to change, can change level immediately. */
1483                 re->level = info->new_level;
1484                 re->backup_blocks = 0;
1485                 return NULL;
1486         }
1487         if (re->after.data_disks == 1 && re->before.data_disks == 1) {
1488                 /* chunk and layout changes make no difference */
1489                 re->level = info->new_level;
1490                 re->backup_blocks = 0;
1491                 return NULL;
1492         }
1493
1494         if (re->after.data_disks == re->before.data_disks &&
1495             get_linux_version() < 2006032)
1496                 return "in-place reshape is not safe before 2.6.32 - sorry.";
1497
1498         if (re->after.data_disks < re->before.data_disks &&
1499             get_linux_version() < 2006030)
1500                 return "reshape to fewer devices is not supported before 2.6.30 - sorry.";
1501
1502         re->backup_blocks = compute_backup_blocks(
1503                 info->new_chunk, info->array.chunk_size,
1504                 re->after.data_disks,
1505                 re->before.data_disks);
1506         re->min_offset_change = re->backup_blocks / re->before.data_disks;
1507
1508         re->new_size = info->component_size * re->after.data_disks;
1509         return NULL;
1510 }
1511
1512 static int set_array_size(struct supertype *st, struct mdinfo *sra,
1513                           char *text_version)
1514 {
1515         struct mdinfo *info;
1516         char *subarray;
1517         int ret_val = -1;
1518
1519         if ((st == NULL) || (sra == NULL))
1520                 return ret_val;
1521
1522         if (text_version == NULL)
1523                 text_version = sra->text_version;
1524         subarray = strchr(text_version+1, '/')+1;
1525         info = st->ss->container_content(st, subarray);
1526         if (info) {
1527                 unsigned long long current_size = 0;
1528                 unsigned long long new_size =
1529                         info->custom_array_size/2;
1530
1531                 if (sysfs_get_ll(sra, NULL, "array_size", &current_size) == 0 &&
1532                     new_size > current_size) {
1533                         if (sysfs_set_num(sra, NULL, "array_size", new_size)
1534                                         < 0)
1535                                 dprintf("Error: Cannot set array size");
1536                         else {
1537                                 ret_val = 0;
1538                                 dprintf("Array size changed");
1539                         }
1540                         dprintf_cont(" from %llu to %llu.\n",
1541                                      current_size, new_size);
1542                 }
1543                 sysfs_free(info);
1544         } else
1545                 dprintf("Error: set_array_size(): info pointer in NULL\n");
1546
1547         return ret_val;
1548 }
1549
1550 static int reshape_array(char *container, int fd, char *devname,
1551                          struct supertype *st, struct mdinfo *info,
1552                          int force, struct mddev_dev *devlist,
1553                          unsigned long long data_offset,
1554                          char *backup_file, int verbose, int forked,
1555                          int restart, int freeze_reshape);
1556 static int reshape_container(char *container, char *devname,
1557                              int mdfd,
1558                              struct supertype *st,
1559                              struct mdinfo *info,
1560                              int force,
1561                              char *backup_file, int verbose,
1562                              int forked, int restart, int freeze_reshape);
1563
1564 int Grow_reshape(char *devname, int fd,
1565                  struct mddev_dev *devlist,
1566                  unsigned long long data_offset,
1567                  struct context *c, struct shape *s)
1568 {
1569         /* Make some changes in the shape of an array.
1570          * The kernel must support the change.
1571          *
1572          * There are three different changes.  Each can trigger
1573          * a resync or recovery so we freeze that until we have
1574          * requested everything (if kernel supports freezing - 2.6.30).
1575          * The steps are:
1576          *  - change size (i.e. component_size)
1577          *  - change level
1578          *  - change layout/chunksize/ndisks
1579          *
1580          * The last can require a reshape.  It is different on different
1581          * levels so we need to check the level before actioning it.
1582          * Some times the level change needs to be requested after the
1583          * reshape (e.g. raid6->raid5, raid5->raid0)
1584          *
1585          */
1586         struct mdu_array_info_s array;
1587         int rv = 0;
1588         struct supertype *st;
1589         char *subarray = NULL;
1590
1591         int frozen;
1592         int changed = 0;
1593         char *container = NULL;
1594         int cfd = -1;
1595
1596         struct mddev_dev *dv;
1597         int added_disks;
1598
1599         struct mdinfo info;
1600         struct mdinfo *sra;
1601
1602         if (ioctl(fd, GET_ARRAY_INFO, &array) < 0) {
1603                 pr_err("%s is not an active md array - aborting\n",
1604                         devname);
1605                 return 1;
1606         }
1607         if (data_offset != INVALID_SECTORS && array.level != 10
1608             && (array.level < 4 || array.level > 6)) {
1609                 pr_err("--grow --data-offset not yet supported\n");
1610                 return 1;
1611         }
1612
1613         if (s->size > 0 &&
1614             (s->chunk || s->level!= UnSet || s->layout_str || s->raiddisks)) {
1615                 pr_err("cannot change component size at the same time as other changes.\n"
1616                         "   Change size first, then check data is intact before making other changes.\n");
1617                 return 1;
1618         }
1619
1620         if (s->raiddisks && s->raiddisks < array.raid_disks && array.level > 1 &&
1621             get_linux_version() < 2006032 &&
1622             !check_env("MDADM_FORCE_FEWER")) {
1623                 pr_err("reducing the number of devices is not safe before Linux 2.6.32\n"
1624                         "       Please use a newer kernel\n");
1625                 return 1;
1626         }
1627
1628         st = super_by_fd(fd, &subarray);
1629         if (!st) {
1630                 pr_err("Unable to determine metadata format for %s\n", devname);
1631                 return 1;
1632         }
1633         if (s->raiddisks > st->max_devs) {
1634                 pr_err("Cannot increase raid-disks on this array beyond %d\n", st->max_devs);
1635                 return 1;
1636         }
1637         if (s->level == 0 &&
1638             (array.state & (1<<MD_SB_BITMAP_PRESENT)) &&
1639             !(array.state & (1<<MD_SB_CLUSTERED))) {
1640                 array.state &= ~(1<<MD_SB_BITMAP_PRESENT);
1641                 if (ioctl(fd, SET_ARRAY_INFO, &array)!= 0) {
1642                         pr_err("failed to remove internal bitmap.\n");
1643                         return 1;
1644                 }
1645         }
1646
1647         /* in the external case we need to check that the requested reshape is
1648          * supported, and perform an initial check that the container holds the
1649          * pre-requisite spare devices (mdmon owns final validation)
1650          */
1651         if (st->ss->external) {
1652                 int rv;
1653
1654                 if (subarray) {
1655                         container = st->container_devnm;
1656                         cfd = open_dev_excl(st->container_devnm);
1657                 } else {
1658                         container = st->devnm;
1659                         close(fd);
1660                         cfd = open_dev_excl(st->devnm);
1661                         fd = cfd;
1662                 }
1663                 if (cfd < 0) {
1664                         pr_err("Unable to open container for %s\n",
1665                                 devname);
1666                         free(subarray);
1667                         return 1;
1668                 }
1669
1670                 rv = st->ss->load_container(st, cfd, NULL);
1671
1672                 if (rv) {
1673                         pr_err("Cannot read superblock for %s\n",
1674                                 devname);
1675                         free(subarray);
1676                         return 1;
1677                 }
1678
1679                 /* check if operation is supported for metadata handler */
1680                 if (st->ss->container_content) {
1681                         struct mdinfo *cc = NULL;
1682                         struct mdinfo *content = NULL;
1683
1684                         cc = st->ss->container_content(st, subarray);
1685                         for (content = cc; content ; content = content->next) {
1686                                 int allow_reshape = 1;
1687
1688                                 /* check if reshape is allowed based on metadata
1689                                  * indications stored in content.array.status
1690                                  */
1691                                 if (content->array.state & (1<<MD_SB_BLOCK_VOLUME))
1692                                         allow_reshape = 0;
1693                                 if (content->array.state
1694                                     & (1<<MD_SB_BLOCK_CONTAINER_RESHAPE))
1695                                         allow_reshape = 0;
1696                                 if (!allow_reshape) {
1697                                         pr_err("cannot reshape arrays in container with unsupported metadata: %s(%s)\n",
1698                                                devname, container);
1699                                         sysfs_free(cc);
1700                                         free(subarray);
1701                                         return 1;
1702                                 }
1703                         }
1704                         sysfs_free(cc);
1705                 }
1706                 if (mdmon_running(container))
1707                         st->update_tail = &st->updates;
1708         }
1709
1710         added_disks = 0;
1711         for (dv = devlist; dv; dv = dv->next)
1712                 added_disks++;
1713         if (s->raiddisks > array.raid_disks &&
1714             array.spare_disks +added_disks < (s->raiddisks - array.raid_disks) &&
1715             !c->force) {
1716                 pr_err("Need %d spare%s to avoid degraded array, and only have %d.\n"
1717                        "       Use --force to over-ride this check.\n",
1718                        s->raiddisks - array.raid_disks,
1719                        s->raiddisks - array.raid_disks == 1 ? "" : "s",
1720                        array.spare_disks + added_disks);
1721                 return 1;
1722         }
1723
1724         sra = sysfs_read(fd, NULL, GET_LEVEL | GET_DISKS | GET_DEVS
1725                          | GET_STATE | GET_VERSION);
1726         if (sra) {
1727                 if (st->ss->external && subarray == NULL) {
1728                         array.level = LEVEL_CONTAINER;
1729                         sra->array.level = LEVEL_CONTAINER;
1730                 }
1731         } else {
1732                 pr_err("failed to read sysfs parameters for %s\n",
1733                         devname);
1734                 return 1;
1735         }
1736         frozen = freeze(st);
1737         if (frozen < -1) {
1738                 /* freeze() already spewed the reason */
1739                 sysfs_free(sra);
1740                 return 1;
1741         } else if (frozen < 0) {
1742                 pr_err("%s is performing resync/recovery and cannot be reshaped\n", devname);
1743                 sysfs_free(sra);
1744                 return 1;
1745         }
1746
1747         /* ========= set size =============== */
1748         if (s->size > 0 && (s->size == MAX_SIZE || s->size != (unsigned)array.size)) {
1749                 unsigned long long orig_size = get_component_size(fd)/2;
1750                 unsigned long long min_csize;
1751                 struct mdinfo *mdi;
1752                 int raid0_takeover = 0;
1753
1754                 if (orig_size == 0)
1755                         orig_size = (unsigned) array.size;
1756
1757                 if (orig_size == 0) {
1758                         pr_err("Cannot set device size in this type of array.\n");
1759                         rv = 1;
1760                         goto release;
1761                 }
1762
1763                 if (reshape_super(st, s->size, UnSet, UnSet, 0, 0, UnSet, NULL,
1764                                   devname, APPLY_METADATA_CHANGES, c->verbose > 0)) {
1765                         rv = 1;
1766                         goto release;
1767                 }
1768                 sync_metadata(st);
1769                 if (st->ss->external) {
1770                         /* metadata can have size limitation
1771                          * update size value according to metadata information
1772                          */
1773                         struct mdinfo *sizeinfo =
1774                                 st->ss->container_content(st, subarray);
1775                         if (sizeinfo) {
1776                                 unsigned long long new_size =
1777                                         sizeinfo->custom_array_size/2;
1778                                 int data_disks = get_data_disks(
1779                                                 sizeinfo->array.level,
1780                                                 sizeinfo->array.layout,
1781                                                 sizeinfo->array.raid_disks);
1782                                 new_size /= data_disks;
1783                                 dprintf("Metadata size correction from %llu to %llu (%llu)\n", orig_size, new_size,
1784                                         new_size * data_disks);
1785                                 s->size = new_size;
1786                                 sysfs_free(sizeinfo);
1787                         }
1788                 }
1789
1790                 /* Update the size of each member device in case
1791                  * they have been resized.  This will never reduce
1792                  * below the current used-size.  The "size" attribute
1793                  * understands '0' to mean 'max'.
1794                  */
1795                 min_csize = 0;
1796                 rv = 0;
1797                 for (mdi = sra->devs; mdi; mdi = mdi->next) {
1798                         if (sysfs_set_num(sra, mdi, "size",
1799                                           s->size == MAX_SIZE ? 0 : s->size) < 0) {
1800                                 /* Probably kernel refusing to let us
1801                                  * reduce the size - not an error.
1802                                  */
1803                                 break;
1804                         }
1805                         if (array.not_persistent == 0 &&
1806                             array.major_version == 0 &&
1807                             get_linux_version() < 3001000) {
1808                                 /* Dangerous to allow size to exceed 2TB */
1809                                 unsigned long long csize;
1810                                 if (sysfs_get_ll(sra, mdi, "size", &csize) == 0) {
1811                                         if (csize >= 2ULL*1024*1024*1024)
1812                                                 csize = 2ULL*1024*1024*1024;
1813                                         if ((min_csize == 0 || (min_csize
1814                                                                 > csize)))
1815                                                 min_csize = csize;
1816                                 }
1817                         }
1818                 }
1819                 if (rv) {
1820                         pr_err("Cannot set size on array members.\n");
1821                         goto size_change_error;
1822                 }
1823                 if (min_csize && s->size > min_csize) {
1824                         pr_err("Cannot safely make this array use more than 2TB per device on this kernel.\n");
1825                         rv = 1;
1826                         goto size_change_error;
1827                 }
1828                 if (min_csize && s->size == MAX_SIZE) {
1829                         /* Don't let the kernel choose a size - it will get
1830                          * it wrong
1831                          */
1832                         pr_err("Limited v0.90 array to 2TB per device\n");
1833                         s->size = min_csize;
1834                 }
1835                 if (st->ss->external) {
1836                         if (sra->array.level == 0) {
1837                                 rv = sysfs_set_str(sra, NULL, "level",
1838                                                    "raid5");
1839                                 if (!rv) {
1840                                         raid0_takeover = 1;
1841                                         /* get array parameters after takeover
1842                                          * to change one parameter at time only
1843                                          */
1844                                         rv = ioctl(fd, GET_ARRAY_INFO, &array);
1845                                 }
1846                         }
1847                         /* make sure mdmon is
1848                          * aware of the new level */
1849                         if (!mdmon_running(st->container_devnm))
1850                                 start_mdmon(st->container_devnm);
1851                         ping_monitor(container);
1852                         if (mdmon_running(st->container_devnm) &&
1853                                         st->update_tail == NULL)
1854                                 st->update_tail = &st->updates;
1855                 }
1856
1857                 if (s->size == MAX_SIZE)
1858                         s->size = 0;
1859                 array.size = s->size;
1860                 if (s->size & ~INT32_MAX) {
1861                         /* got truncated to 32bit, write to
1862                          * component_size instead
1863                          */
1864                         if (sra)
1865                                 rv = sysfs_set_num(sra, NULL,
1866                                                    "component_size", s->size);
1867                         else
1868                                 rv = -1;
1869                 } else {
1870                         rv = ioctl(fd, SET_ARRAY_INFO, &array);
1871
1872                         /* manage array size when it is managed externally
1873                          */
1874                         if ((rv == 0) && st->ss->external)
1875                                 rv = set_array_size(st, sra, sra->text_version);
1876                 }
1877
1878                 if (raid0_takeover) {
1879                         /* do not recync non-existing parity,
1880                          * we will drop it anyway
1881                          */
1882                         sysfs_set_str(sra, NULL, "sync_action", "frozen");
1883                         /* go back to raid0, drop parity disk
1884                          */
1885                         sysfs_set_str(sra, NULL, "level", "raid0");
1886                         ioctl(fd, GET_ARRAY_INFO, &array);
1887                 }
1888
1889 size_change_error:
1890                 if (rv != 0) {
1891                         int err = errno;
1892
1893                         /* restore metadata */
1894                         if (reshape_super(st, orig_size, UnSet, UnSet, 0, 0,
1895                                           UnSet, NULL, devname,
1896                                           ROLLBACK_METADATA_CHANGES,
1897                                           c->verbose) == 0)
1898                                 sync_metadata(st);
1899                         pr_err("Cannot set device size for %s: %s\n",
1900                                 devname, strerror(err));
1901                         if (err == EBUSY &&
1902                             (array.state & (1<<MD_SB_BITMAP_PRESENT)))
1903                                 cont_err("Bitmap must be removed before size can be changed\n");
1904                         rv = 1;
1905                         goto release;
1906                 }
1907                 if (s->assume_clean) {
1908                         /* This will fail on kernels older than 3.0 unless
1909                          * a backport has been arranged.
1910                          */
1911                         if (sra == NULL ||
1912                             sysfs_set_str(sra, NULL, "resync_start", "none") < 0)
1913                                 pr_err("--assume-clean not supported with --grow on this kernel\n");
1914                 }
1915                 ioctl(fd, GET_ARRAY_INFO, &array);
1916                 s->size = get_component_size(fd)/2;
1917                 if (s->size == 0)
1918                         s->size = array.size;
1919                 if (c->verbose >= 0) {
1920                         if (s->size == orig_size)
1921                                 pr_err("component size of %s unchanged at %lluK\n",
1922                                         devname, s->size);
1923                         else
1924                                 pr_err("component size of %s has been set to %lluK\n",
1925                                         devname, s->size);
1926                 }
1927                 changed = 1;
1928         } else if (array.level != LEVEL_CONTAINER) {
1929                 s->size = get_component_size(fd)/2;
1930                 if (s->size == 0)
1931                         s->size = array.size;
1932         }
1933
1934         /* See if there is anything else to do */
1935         if ((s->level == UnSet || s->level == array.level) &&
1936             (s->layout_str == NULL) &&
1937             (s->chunk == 0 || s->chunk == array.chunk_size) &&
1938             data_offset == INVALID_SECTORS &&
1939             (s->raiddisks == 0 || s->raiddisks == array.raid_disks)) {
1940                 /* Nothing more to do */
1941                 if (!changed && c->verbose >= 0)
1942                         pr_err("%s: no change requested\n",
1943                                 devname);
1944                 goto release;
1945         }
1946
1947         /* ========= check for Raid10/Raid1 -> Raid0 conversion ===============
1948          * current implementation assumes that following conditions must be met:
1949          * - RAID10:
1950          *      - far_copies == 1
1951          *      - near_copies == 2
1952          */
1953         if ((s->level == 0 && array.level == 10 && sra &&
1954              array.layout == ((1 << 8) + 2) && !(array.raid_disks & 1)) ||
1955             (s->level == 0 && array.level == 1 && sra)) {
1956                 int err;
1957                 err = remove_disks_for_takeover(st, sra, array.layout);
1958                 if (err) {
1959                         dprintf("Array cannot be reshaped\n");
1960                         if (cfd > -1)
1961                                 close(cfd);
1962                         rv = 1;
1963                         goto release;
1964                 }
1965                 /* Make sure mdmon has seen the device removal
1966                  * and updated metadata before we continue with
1967                  * level change
1968                  */
1969                 if (container)
1970                         ping_monitor(container);
1971         }
1972
1973         memset(&info, 0, sizeof(info));
1974         info.array = array;
1975         sysfs_init(&info, fd, NULL);
1976         strcpy(info.text_version, sra->text_version);
1977         info.component_size = s->size*2;
1978         info.new_level = s->level;
1979         info.new_chunk = s->chunk * 1024;
1980         if (info.array.level == LEVEL_CONTAINER) {
1981                 info.delta_disks = UnSet;
1982                 info.array.raid_disks = s->raiddisks;
1983         } else if (s->raiddisks)
1984                 info.delta_disks = s->raiddisks - info.array.raid_disks;
1985         else
1986                 info.delta_disks = UnSet;
1987         if (s->layout_str == NULL) {
1988                 info.new_layout = UnSet;
1989                 if (info.array.level == 6 &&
1990                     (info.new_level == 6 || info.new_level == UnSet) &&
1991                     info.array.layout >= 16) {
1992                         pr_err("%s has a non-standard layout.  If you wish to preserve this\n", devname);
1993                         cont_err("during the reshape, please specify --layout=preserve\n");
1994                         cont_err("If you want to change it, specify a layout or use --layout=normalise\n");
1995                         rv = 1;
1996                         goto release;
1997                 }
1998         } else if (strcmp(s->layout_str, "normalise") == 0 ||
1999                    strcmp(s->layout_str, "normalize") == 0) {
2000                 /* If we have a -6 RAID6 layout, remove the '-6'. */
2001                 info.new_layout = UnSet;
2002                 if (info.array.level == 6 && info.new_level == UnSet) {
2003                         char l[40], *h;
2004                         strcpy(l, map_num(r6layout, info.array.layout));
2005                         h = strrchr(l, '-');
2006                         if (h && strcmp(h, "-6") == 0) {
2007                                 *h = 0;
2008                                 info.new_layout = map_name(r6layout, l);
2009                         }
2010                 } else {
2011                         pr_err("%s is only meaningful when reshaping a RAID6 array.\n", s->layout_str);
2012                         rv = 1;
2013                         goto release;
2014                 }
2015         } else if (strcmp(s->layout_str, "preserve") == 0) {
2016                 /* This means that a non-standard RAID6 layout
2017                  * is OK.
2018                  * In particular:
2019                  * - When reshape a RAID6 (e.g. adding a device)
2020                  *   which is in a non-standard layout, it is OK
2021                  *   to preserve that layout.
2022                  * - When converting a RAID5 to RAID6, leave it in
2023                  *   the XXX-6 layout, don't re-layout.
2024                  */
2025                 if (info.array.level == 6 && info.new_level == UnSet)
2026                         info.new_layout = info.array.layout;
2027                 else if (info.array.level == 5 && info.new_level == 6) {
2028                         char l[40];
2029                         strcpy(l, map_num(r5layout, info.array.layout));
2030                         strcat(l, "-6");
2031                         info.new_layout = map_name(r6layout, l);
2032                 } else {
2033                         pr_err("%s in only meaningful when reshaping to RAID6\n", s->layout_str);
2034                         rv = 1;
2035                         goto release;
2036                 }
2037         } else {
2038                 int l = info.new_level;
2039                 if (l == UnSet)
2040                         l = info.array.level;
2041                 switch (l) {
2042                 case 5:
2043                         info.new_layout = map_name(r5layout, s->layout_str);
2044                         break;
2045                 case 6:
2046                         info.new_layout = map_name(r6layout, s->layout_str);
2047                         break;
2048                 case 10:
2049                         info.new_layout = parse_layout_10(s->layout_str);
2050                         break;
2051                 case LEVEL_FAULTY:
2052                         info.new_layout = parse_layout_faulty(s->layout_str);
2053                         break;
2054                 default:
2055                         pr_err("layout not meaningful with this level\n");
2056                         rv = 1;
2057                         goto release;
2058                 }
2059                 if (info.new_layout == UnSet) {
2060                         pr_err("layout %s not understood for this level\n",
2061                                 s->layout_str);
2062                         rv = 1;
2063                         goto release;
2064                 }
2065         }
2066
2067         if (array.level == LEVEL_FAULTY) {
2068                 if (s->level != UnSet && s->level != array.level) {
2069                         pr_err("cannot change level of Faulty device\n");
2070                         rv =1 ;
2071                 }
2072                 if (s->chunk) {
2073                         pr_err("cannot set chunksize of Faulty device\n");
2074                         rv =1 ;
2075                 }
2076                 if (s->raiddisks && s->raiddisks != 1) {
2077                         pr_err("cannot set raid_disks of Faulty device\n");
2078                         rv =1 ;
2079                 }
2080                 if (s->layout_str) {
2081                         if (ioctl(fd, GET_ARRAY_INFO, &array) != 0) {
2082                                 dprintf("Cannot get array information.\n");
2083                                 goto release;
2084                         }
2085                         array.layout = info.new_layout;
2086                         if (ioctl(fd, SET_ARRAY_INFO, &array) != 0) {
2087                                 pr_err("failed to set new layout\n");
2088                                 rv = 1;
2089                         } else if (c->verbose >= 0)
2090                                 printf("layout for %s set to %d\n",
2091                                        devname, array.layout);
2092                 }
2093         } else if (array.level == LEVEL_CONTAINER) {
2094                 /* This change is to be applied to every array in the
2095                  * container.  This is only needed when the metadata imposes
2096                  * restraints of the various arrays in the container.
2097                  * Currently we only know that IMSM requires all arrays
2098                  * to have the same number of devices so changing the
2099                  * number of devices (On-Line Capacity Expansion) must be
2100                  * performed at the level of the container
2101                  */
2102                 if (fd > 0) {
2103                         close(fd);
2104                         fd = -1;
2105                 }
2106                 rv = reshape_container(container, devname, -1, st, &info,
2107                                        c->force, c->backup_file, c->verbose, 0, 0, 0);
2108                 frozen = 0;
2109         } else {
2110                 /* get spare devices from external metadata
2111                  */
2112                 if (st->ss->external) {
2113                         struct mdinfo *info2;
2114
2115                         info2 = st->ss->container_content(st, subarray);
2116                         if (info2) {
2117                                 info.array.spare_disks =
2118                                         info2->array.spare_disks;
2119                                 sysfs_free(info2);
2120                         }
2121                 }
2122
2123                 /* Impose these changes on a single array.  First
2124                  * check that the metadata is OK with the change. */
2125
2126                 if (reshape_super(st, 0, info.new_level,
2127                                   info.new_layout, info.new_chunk,
2128                                   info.array.raid_disks, info.delta_disks,
2129                                   c->backup_file, devname, APPLY_METADATA_CHANGES,
2130                                   c->verbose)) {
2131                         rv = 1;
2132                         goto release;
2133                 }
2134                 sync_metadata(st);
2135                 rv = reshape_array(container, fd, devname, st, &info, c->force,
2136                                    devlist, data_offset, c->backup_file, c->verbose,
2137                                    0, 0, 0);
2138                 frozen = 0;
2139         }
2140 release:
2141         sysfs_free(sra);
2142         if (frozen > 0)
2143                 unfreeze(st);
2144         return rv;
2145 }
2146
2147 /* verify_reshape_position()
2148  *      Function checks if reshape position in metadata is not farther
2149  *      than position in md.
2150  * Return value:
2151  *       0 : not valid sysfs entry
2152  *              it can be caused by not started reshape, it should be started
2153  *              by reshape array or raid0 array is before takeover
2154  *      -1 :    error, reshape position is obviously wrong
2155  *       1 :    success, reshape progress correct or updated
2156 */
2157 static int verify_reshape_position(struct mdinfo *info, int level)
2158 {
2159         int ret_val = 0;
2160         char buf[40];
2161         int rv;
2162
2163         /* read sync_max, failure can mean raid0 array */
2164         rv = sysfs_get_str(info, NULL, "sync_max", buf, 40);
2165
2166         if (rv > 0) {
2167                 char *ep;
2168                 unsigned long long position = strtoull(buf, &ep, 0);
2169
2170                 dprintf("Read sync_max sysfs entry is: %s\n", buf);
2171                 if (!(ep == buf || (*ep != 0 && *ep != '\n' && *ep != ' '))) {
2172                         position *= get_data_disks(level,
2173                                                    info->new_layout,
2174                                                    info->array.raid_disks);
2175                         if (info->reshape_progress < position) {
2176                                 dprintf("Corrected reshape progress (%llu) to md position (%llu)\n",
2177                                         info->reshape_progress, position);
2178                                 info->reshape_progress = position;
2179                                 ret_val = 1;
2180                         } else if (info->reshape_progress > position) {
2181                                 pr_err("Fatal error: array reshape was not properly frozen (expected reshape position is %llu, but reshape progress is %llu.\n",
2182                                        position, info->reshape_progress);
2183                                 ret_val = -1;
2184                         } else {
2185                                 dprintf("Reshape position in md and metadata are the same;");
2186                                 ret_val = 1;
2187                         }
2188                 }
2189         } else if (rv == 0) {
2190                 /* for valid sysfs entry, 0-length content
2191                  * should be indicated as error
2192                  */
2193                 ret_val = -1;
2194         }
2195
2196         return ret_val;
2197 }
2198
2199 static unsigned long long choose_offset(unsigned long long lo,
2200                                         unsigned long long hi,
2201                                         unsigned long long min,
2202                                         unsigned long long max)
2203 {
2204         /* Choose a new offset between hi and lo.
2205          * It must be between min and max, but
2206          * we would prefer something near the middle of hi/lo, and also
2207          * prefer to be aligned to a big power of 2.
2208          *
2209          * So we start with the middle, then for each bit,
2210          * starting at '1' and increasing, if it is set, we either
2211          * add it or subtract it if possible, preferring the option
2212          * which is furthest from the boundary.
2213          *
2214          * We stop once we get a 1MB alignment. As units are in sectors,
2215          * 1MB = 2*1024 sectors.
2216          */
2217         unsigned long long choice = (lo + hi) / 2;
2218         unsigned long long bit = 1;
2219
2220         for (bit = 1; bit < 2*1024; bit = bit << 1) {
2221                 unsigned long long bigger, smaller;
2222                 if (! (bit & choice))
2223                         continue;
2224                 bigger = choice + bit;
2225                 smaller = choice - bit;
2226                 if (bigger > max && smaller < min)
2227                         break;
2228                 if (bigger > max)
2229                         choice = smaller;
2230                 else if (smaller < min)
2231                         choice = bigger;
2232                 else if (hi - bigger > smaller - lo)
2233                         choice = bigger;
2234                 else
2235                         choice = smaller;
2236         }
2237         return choice;
2238 }
2239
2240 static int set_new_data_offset(struct mdinfo *sra, struct supertype *st,
2241                                char *devname, int delta_disks,
2242                                unsigned long long data_offset,
2243                                unsigned long long min,
2244                                int can_fallback)
2245 {
2246         struct mdinfo *sd;
2247         int dir = 0;
2248         int err = 0;
2249         unsigned long long before, after;
2250
2251         /* Need to find min space before and after so same is used
2252          * on all devices
2253          */
2254         before = UINT64_MAX;
2255         after = UINT64_MAX;
2256         for (sd = sra->devs; sd; sd = sd->next) {
2257                 char *dn;
2258                 int dfd;
2259                 int rv;
2260                 struct supertype *st2;
2261                 struct mdinfo info2;
2262
2263                 if (sd->disk.state & (1<<MD_DISK_FAULTY))
2264                         continue;
2265                 dn = map_dev(sd->disk.major, sd->disk.minor, 0);
2266                 dfd = dev_open(dn, O_RDONLY);
2267                 if (dfd < 0) {
2268                         pr_err("%s: cannot open component %s\n",
2269                                 devname, dn ? dn : "-unknown-");
2270                         goto release;
2271                 }
2272                 st2 = dup_super(st);
2273                 rv = st2->ss->load_super(st2,dfd, NULL);
2274                 close(dfd);
2275                 if (rv) {
2276                         free(st2);
2277                         pr_err("%s: cannot get superblock from %s\n",
2278                                 devname, dn);
2279                         goto release;
2280                 }
2281                 st2->ss->getinfo_super(st2, &info2, NULL);
2282                 st2->ss->free_super(st2);
2283                 free(st2);
2284                 if (info2.space_before == 0 &&
2285                     info2.space_after == 0) {
2286                         /* Metadata doesn't support data_offset changes */
2287                         if (!can_fallback)
2288                                 pr_err("%s: Metadata version doesn't support data_offset changes\n",
2289                                        devname);
2290                         goto fallback;
2291                 }
2292                 if (before > info2.space_before)
2293                         before = info2.space_before;
2294                 if (after > info2.space_after)
2295                         after = info2.space_after;
2296
2297                 if (data_offset != INVALID_SECTORS) {
2298                         if (dir == 0) {
2299                                 if (info2.data_offset == data_offset) {
2300                                         pr_err("%s: already has that data_offset\n",
2301                                                dn);
2302                                         goto release;
2303                                 }
2304                                 if (data_offset < info2.data_offset)
2305                                         dir = -1;
2306                                 else
2307                                         dir = 1;
2308                         } else if ((data_offset <= info2.data_offset && dir == 1) ||
2309                                    (data_offset >= info2.data_offset && dir == -1)) {
2310                                 pr_err("%s: differing data offsets on devices make this --data-offset setting impossible\n",
2311                                         dn);
2312                                 goto release;
2313                         }
2314                 }
2315         }
2316         if (before == UINT64_MAX)
2317                 /* impossible really, there must be no devices */
2318                 return 1;
2319
2320         for (sd = sra->devs; sd; sd = sd->next) {
2321                 char *dn = map_dev(sd->disk.major, sd->disk.minor, 0);
2322                 unsigned long long new_data_offset;
2323
2324                 if (sd->disk.state & (1<<MD_DISK_FAULTY))
2325                         continue;
2326                 if (delta_disks < 0) {
2327                         /* Don't need any space as array is shrinking
2328                          * just move data_offset up by min
2329                          */
2330                         if (data_offset == INVALID_SECTORS)
2331                                 new_data_offset = sd->data_offset + min;
2332                         else {
2333                                 if (data_offset < sd->data_offset + min) {
2334                                         pr_err("--data-offset too small for %s\n",
2335                                                 dn);
2336                                         goto release;
2337                                 }
2338                                 new_data_offset = data_offset;
2339                         }
2340                 } else if (delta_disks > 0) {
2341                         /* need space before */
2342                         if (before < min) {
2343                                 if (can_fallback)
2344                                         goto fallback;
2345                                 pr_err("Insufficient head-space for reshape on %s\n",
2346                                         dn);
2347                                 goto release;
2348                         }
2349                         if (data_offset == INVALID_SECTORS)
2350                                 new_data_offset = sd->data_offset - min;
2351                         else {
2352                                 if (data_offset > sd->data_offset - min) {
2353                                         pr_err("--data-offset too large for %s\n",
2354                                                 dn);
2355                                         goto release;
2356                                 }
2357                                 new_data_offset = data_offset;
2358                         }
2359                 } else {
2360                         if (dir == 0) {
2361                                 /* can move up or down.  If 'data_offset'
2362                                  * was set we would have already decided,
2363                                  * so just choose direction with most space.
2364                                  */
2365                                 if (before > after)
2366                                         dir = -1;
2367                                 else
2368                                         dir = 1;
2369                         }
2370                         sysfs_set_str(sra, NULL, "reshape_direction",
2371                                       dir == 1 ? "backwards" : "forwards");
2372                         if (dir > 0) {
2373                                 /* Increase data offset */
2374                                 if (after < min) {
2375                                         if (can_fallback)
2376                                                 goto fallback;
2377                                         pr_err("Insufficient tail-space for reshape on %s\n",
2378                                                 dn);
2379                                         goto release;
2380                                 }
2381                                 if (data_offset != INVALID_SECTORS &&
2382                                     data_offset < sd->data_offset + min) {
2383                                         pr_err("--data-offset too small on %s\n",
2384                                                 dn);
2385                                         goto release;
2386                                 }
2387                                 if (data_offset != INVALID_SECTORS)
2388                                         new_data_offset = data_offset;
2389                                 else
2390                                         new_data_offset = choose_offset(sd->data_offset,
2391                                                                         sd->data_offset + after,
2392                                                                         sd->data_offset + min,
2393                                                                         sd->data_offset + after);
2394                         } else {
2395                                 /* Decrease data offset */
2396                                 if (before < min) {
2397                                         if (can_fallback)
2398                                                 goto fallback;
2399                                         pr_err("insufficient head-room on %s\n",
2400                                                 dn);
2401                                         goto release;
2402                                 }
2403                                 if (data_offset != INVALID_SECTORS &&
2404                                     data_offset < sd->data_offset - min) {
2405                                         pr_err("--data-offset too small on %s\n",
2406                                                 dn);
2407                                         goto release;
2408                                 }
2409                                 if (data_offset != INVALID_SECTORS)
2410                                         new_data_offset = data_offset;
2411                                 else
2412                                         new_data_offset = choose_offset(sd->data_offset - before,
2413                                                                         sd->data_offset,
2414                                                                         sd->data_offset - before,
2415                                                                         sd->data_offset - min);
2416                         }
2417                 }
2418                 err = sysfs_set_num(sra, sd, "new_offset", new_data_offset);
2419                 if (err < 0 && errno == E2BIG) {
2420                         /* try again after increasing data size to max */
2421                         err = sysfs_set_num(sra, sd, "size", 0);
2422                         if (err < 0 && errno == EINVAL &&
2423                             !(sd->disk.state & (1<<MD_DISK_SYNC))) {
2424                                 /* some kernels have a bug where you cannot
2425                                  * use '0' on spare devices. */
2426                                 sysfs_set_num(sra, sd, "size",
2427                                               (sra->component_size + after)/2);
2428                         }
2429                         err = sysfs_set_num(sra, sd, "new_offset",
2430                                             new_data_offset);
2431                 }
2432                 if (err < 0) {
2433                         if (errno == E2BIG && data_offset != INVALID_SECTORS) {
2434                                 pr_err("data-offset is too big for %s\n",
2435                                        dn);
2436                                 goto release;
2437                         }
2438                         if (sd == sra->devs &&
2439                             (errno == ENOENT || errno == E2BIG))
2440                                 /* Early kernel, no 'new_offset' file,
2441                                  * or kernel doesn't like us.
2442                                  * For RAID5/6 this is not fatal
2443                                  */
2444                                 return 1;
2445                         pr_err("Cannot set new_offset for %s\n",
2446                                 dn);
2447                         break;
2448                 }
2449         }
2450         return err;
2451 release:
2452         return -1;
2453 fallback:
2454         /* Just use a backup file */
2455         return 1;
2456 }
2457
2458 static int raid10_reshape(char *container, int fd, char *devname,
2459                           struct supertype *st, struct mdinfo *info,
2460                           struct reshape *reshape,
2461                           unsigned long long data_offset,
2462                           int force, int verbose)
2463 {
2464         /* Changing raid_disks, layout, chunksize or possibly
2465          * just data_offset for a RAID10.
2466          * We must always change data_offset.  We change by at least
2467          * ->min_offset_change which is the largest of the old and new
2468          * chunk sizes.
2469          * If raid_disks is increasing, then data_offset must decrease
2470          * by at least this copy size.
2471          * If raid_disks is unchanged, data_offset must increase or
2472          * decrease by at least min_offset_change but preferably by much more.
2473          * We choose half of the available space.
2474          * If raid_disks is decreasing, data_offset must increase by
2475          * at least min_offset_change.  To allow of this, component_size
2476          * must be decreased by the same amount.
2477          *
2478          * So we calculate the required minimum and direction, possibly
2479          * reduce the component_size, then iterate through the devices
2480          * and set the new_data_offset.
2481          * If that all works, we set chunk_size, layout, raid_disks, and start
2482          * 'reshape'
2483          */
2484         struct mdinfo *sra;
2485         unsigned long long min;
2486         int err = 0;
2487
2488         sra = sysfs_read(fd, NULL,
2489                          GET_COMPONENT|GET_DEVS|GET_OFFSET|GET_STATE|GET_CHUNK
2490                 );
2491         if (!sra) {
2492                 pr_err("%s: Cannot get array details from sysfs\n",
2493                         devname);
2494                 goto release;
2495         }
2496         min = reshape->min_offset_change;
2497
2498         if (info->delta_disks)
2499                 sysfs_set_str(sra, NULL, "reshape_direction",
2500                               info->delta_disks < 0 ? "backwards" : "forwards");
2501         if (info->delta_disks < 0 &&
2502             info->space_after < min) {
2503                 int rv = sysfs_set_num(sra, NULL, "component_size",
2504                                        (sra->component_size -
2505                                         min)/2);
2506                 if (rv) {
2507                         pr_err("cannot reduce component size\n");
2508                         goto release;
2509                 }
2510         }
2511         err = set_new_data_offset(sra, st, devname, info->delta_disks, data_offset,
2512                                   min, 0);
2513         if (err == 1) {
2514                 pr_err("Cannot set new_data_offset: RAID10 reshape not\n");
2515                 cont_err("supported on this kernel\n");
2516                 err = -1;
2517         }
2518         if (err < 0)
2519                 goto release;
2520
2521         if (!err && sysfs_set_num(sra, NULL, "chunk_size", info->new_chunk) < 0)
2522                 err = errno;
2523         if (!err && sysfs_set_num(sra, NULL, "layout", reshape->after.layout) < 0)
2524                 err = errno;
2525         if (!err && sysfs_set_num(sra, NULL, "raid_disks",
2526                                   info->array.raid_disks + info->delta_disks) < 0)
2527                 err = errno;
2528         if (!err && sysfs_set_str(sra, NULL, "sync_action", "reshape") < 0)
2529                 err = errno;
2530         if (err) {
2531                 pr_err("Cannot set array shape for %s\n",
2532                        devname);
2533                 if (err == EBUSY &&
2534                     (info->array.state & (1<<MD_SB_BITMAP_PRESENT)))
2535                         cont_err("       Bitmap must be removed before shape can be changed\n");
2536                 goto release;
2537         }
2538         sysfs_free(sra);
2539         return 0;
2540 release:
2541         sysfs_free(sra);
2542         return 1;
2543 }
2544
2545 static void get_space_after(int fd, struct supertype *st, struct mdinfo *info)
2546 {
2547         struct mdinfo *sra, *sd;
2548         /* Initialisation to silence compiler warning */
2549         unsigned long long min_space_before = 0, min_space_after = 0;
2550         int first = 1;
2551
2552         sra = sysfs_read(fd, NULL, GET_DEVS);
2553         if (!sra)
2554                 return;
2555         for (sd = sra->devs; sd; sd = sd->next) {
2556                 char *dn;
2557                 int dfd;
2558                 struct supertype *st2;
2559                 struct mdinfo info2;
2560
2561                 if (sd->disk.state & (1<<MD_DISK_FAULTY))
2562                         continue;
2563                 dn = map_dev(sd->disk.major, sd->disk.minor, 0);
2564                 dfd = dev_open(dn, O_RDONLY);
2565                 if (dfd < 0)
2566                         break;
2567                 st2 = dup_super(st);
2568                 if (st2->ss->load_super(st2,dfd, NULL)) {
2569                         close(dfd);
2570                         free(st2);
2571                         break;
2572                 }
2573                 close(dfd);
2574                 st2->ss->getinfo_super(st2, &info2, NULL);
2575                 st2->ss->free_super(st2);
2576                 free(st2);
2577                 if (first ||
2578                     min_space_before > info2.space_before)
2579                         min_space_before = info2.space_before;
2580                 if (first ||
2581                     min_space_after > info2.space_after)
2582                         min_space_after = info2.space_after;
2583                 first = 0;
2584         }
2585         if (sd == NULL && !first) {
2586                 info->space_after = min_space_after;
2587                 info->space_before = min_space_before;
2588         }
2589         sysfs_free(sra);
2590 }
2591
2592 static void update_cache_size(char *container, struct mdinfo *sra,
2593                               struct mdinfo *info,
2594                               int disks, unsigned long long blocks)
2595 {
2596         /* Check that the internal stripe cache is
2597          * large enough, or it won't work.
2598          * It must hold at least 4 stripes of the larger
2599          * chunk size
2600          */
2601         unsigned long cache;
2602         cache = max(info->array.chunk_size, info->new_chunk);
2603         cache *= 4; /* 4 stripes minimum */
2604         cache /= 512; /* convert to sectors */
2605         /* make sure there is room for 'blocks' with a bit to spare */
2606         if (cache < 16 + blocks / disks)
2607                 cache = 16 + blocks / disks;
2608         cache /= (4096/512); /* Convert from sectors to pages */
2609
2610         if (sra->cache_size < cache)
2611                 subarray_set_num(container, sra, "stripe_cache_size",
2612                                  cache+1);
2613 }
2614
2615 static int impose_reshape(struct mdinfo *sra,
2616                           struct mdinfo *info,
2617                           struct supertype *st,
2618                           int fd,
2619                           int restart,
2620                           char *devname, char *container,
2621                           struct reshape *reshape)
2622 {
2623         struct mdu_array_info_s array;
2624
2625         sra->new_chunk = info->new_chunk;
2626
2627         if (restart) {
2628                 /* for external metadata checkpoint saved by mdmon can be lost
2629                  * or missed /due to e.g. crash/. Check if md is not during
2630                  * restart farther than metadata points to.
2631                  * If so, this means metadata information is obsolete.
2632                  */
2633                 if (st->ss->external)
2634                         verify_reshape_position(info, reshape->level);
2635                 sra->reshape_progress = info->reshape_progress;
2636         } else {
2637                 sra->reshape_progress = 0;
2638                 if (reshape->after.data_disks < reshape->before.data_disks)
2639                         /* start from the end of the new array */
2640                         sra->reshape_progress = (sra->component_size
2641                                                  * reshape->after.data_disks);
2642         }
2643
2644         ioctl(fd, GET_ARRAY_INFO, &array);
2645         if (info->array.chunk_size == info->new_chunk &&
2646             reshape->before.layout == reshape->after.layout &&
2647             st->ss->external == 0) {
2648                 /* use SET_ARRAY_INFO but only if reshape hasn't started */
2649                 array.raid_disks = reshape->after.data_disks + reshape->parity;
2650                 if (!restart &&
2651                     ioctl(fd, SET_ARRAY_INFO, &array) != 0) {
2652                         int err = errno;
2653
2654                         pr_err("Cannot set device shape for %s: %s\n",
2655                                devname, strerror(errno));
2656
2657                         if (err == EBUSY &&
2658                             (array.state & (1<<MD_SB_BITMAP_PRESENT)))
2659                                 cont_err("Bitmap must be removed before shape can be changed\n");
2660
2661                         goto release;
2662                 }
2663         } else if (!restart) {
2664                 /* set them all just in case some old 'new_*' value
2665                  * persists from some earlier problem.
2666                  */
2667                 int err = 0;
2668                 if (sysfs_set_num(sra, NULL, "chunk_size", info->new_chunk) < 0)
2669                         err = errno;
2670                 if (!err && sysfs_set_num(sra, NULL, "layout",
2671                                           reshape->after.layout) < 0)
2672                         err = errno;
2673                 if (!err && subarray_set_num(container, sra, "raid_disks",
2674                                              reshape->after.data_disks +
2675                                              reshape->parity) < 0)
2676                         err = errno;
2677                 if (err) {
2678                         pr_err("Cannot set device shape for %s\n",
2679                                 devname);
2680
2681                         if (err == EBUSY &&
2682                             (array.state & (1<<MD_SB_BITMAP_PRESENT)))
2683                                 cont_err("Bitmap must be removed before shape can be changed\n");
2684                         goto release;
2685                 }
2686         }
2687         return 0;
2688 release:
2689         return -1;
2690 }
2691
2692 static int impose_level(int fd, int level, char *devname, int verbose)
2693 {
2694         char *c;
2695         struct mdu_array_info_s array;
2696         struct mdinfo info;
2697         sysfs_init(&info, fd, NULL);
2698
2699         ioctl(fd, GET_ARRAY_INFO, &array);
2700         if (level == 0 &&
2701             (array.level >= 4 && array.level <= 6)) {
2702                 /* To convert to RAID0 we need to fail and
2703                  * remove any non-data devices. */
2704                 int found = 0;
2705                 int d;
2706                 int data_disks = array.raid_disks - 1;
2707                 if (array.level == 6)
2708                         data_disks -= 1;
2709                 if (array.level == 5 &&
2710                     array.layout != ALGORITHM_PARITY_N)
2711                         return -1;
2712                 if (array.level == 6 &&
2713                     array.layout != ALGORITHM_PARITY_N_6)
2714                         return -1;
2715                 sysfs_set_str(&info, NULL,"sync_action", "idle");
2716                 /* First remove any spares so no recovery starts */
2717                 for (d = 0, found = 0;
2718                      d < MAX_DISKS && found < array.nr_disks;
2719                      d++) {
2720                         mdu_disk_info_t disk;
2721                         disk.number = d;
2722                         if (ioctl(fd, GET_DISK_INFO, &disk) < 0)
2723                                 continue;
2724                         if (disk.major == 0 && disk.minor == 0)
2725                                 continue;
2726                         found++;
2727                         if ((disk.state & (1 << MD_DISK_ACTIVE))
2728                             && disk.raid_disk < data_disks)
2729                                 /* keep this */
2730                                 continue;
2731                         ioctl(fd, HOT_REMOVE_DISK,
2732                               makedev(disk.major, disk.minor));
2733                 }
2734                 /* Now fail anything left */
2735                 ioctl(fd, GET_ARRAY_INFO, &array);
2736                 for (d = 0, found = 0;
2737                      d < MAX_DISKS && found < array.nr_disks;
2738                      d++) {
2739                         int cnt;
2740                         mdu_disk_info_t disk;
2741                         disk.number = d;
2742                         if (ioctl(fd, GET_DISK_INFO, &disk) < 0)
2743                                 continue;
2744                         if (disk.major == 0 && disk.minor == 0)
2745                                 continue;
2746                         found++;
2747                         if ((disk.state & (1 << MD_DISK_ACTIVE))
2748                             && disk.raid_disk < data_disks)
2749                                 /* keep this */
2750                                 continue;
2751                         ioctl(fd, SET_DISK_FAULTY,
2752                               makedev(disk.major, disk.minor));
2753                         cnt = 5;
2754                         while (ioctl(fd, HOT_REMOVE_DISK,
2755                                      makedev(disk.major, disk.minor)) < 0
2756                                && errno == EBUSY
2757                                && cnt--) {
2758                                 usleep(10000);
2759                         }
2760                 }
2761         }
2762         c = map_num(pers, level);
2763         if (c) {
2764                 int err = sysfs_set_str(&info, NULL, "level", c);
2765                 if (err) {
2766                         err = errno;
2767                         pr_err("%s: could not set level to %s\n",
2768                                 devname, c);
2769                         if (err == EBUSY &&
2770                             (array.state & (1<<MD_SB_BITMAP_PRESENT)))
2771                                 cont_err("Bitmap must be removed before level can be changed\n");
2772                         return err;
2773                 }
2774                 if (verbose >= 0)
2775                         pr_err("level of %s changed to %s\n",
2776                                 devname, c);
2777         }
2778         return 0;
2779 }
2780
2781 int sigterm = 0;
2782 static void catch_term(int sig)
2783 {
2784         sigterm = 1;
2785 }
2786
2787 static int continue_via_systemd(char *devnm)
2788 {
2789         int skipped, i, pid, status;
2790         char pathbuf[1024];
2791         /* In a systemd/udev world, it is best to get systemd to
2792          * run "mdadm --grow --continue" rather than running in the
2793          * background.
2794          */
2795         switch(fork()) {
2796         case  0:
2797                 /* FIXME yuk. CLOSE_EXEC?? */
2798                 skipped = 0;
2799                 for (i = 3; skipped < 20; i++)
2800                         if (close(i) < 0)
2801                                 skipped++;
2802                         else
2803                                 skipped = 0;
2804
2805                 /* Don't want to see error messages from
2806                  * systemctl.  If the service doesn't exist,
2807                  * we fork ourselves.
2808                  */
2809                 close(2);
2810                 open("/dev/null", O_WRONLY);
2811                 snprintf(pathbuf, sizeof(pathbuf), "mdadm-grow-continue@%s.service",
2812                          devnm);
2813                 status = execl("/usr/bin/systemctl", "systemctl",
2814                                "start",
2815                                pathbuf, NULL);
2816                 status = execl("/bin/systemctl", "systemctl", "start",
2817                                pathbuf, NULL);
2818                 exit(1);
2819         case -1: /* Just do it ourselves. */
2820                 break;
2821         default: /* parent - good */
2822                 pid = wait(&status);
2823                 if (pid >= 0 && status == 0)
2824                         return 1;
2825         }
2826         return 0;
2827 }
2828
2829 static int reshape_array(char *container, int fd, char *devname,
2830                          struct supertype *st, struct mdinfo *info,
2831                          int force, struct mddev_dev *devlist,
2832                          unsigned long long data_offset,
2833                          char *backup_file, int verbose, int forked,
2834                          int restart, int freeze_reshape)
2835 {
2836         struct reshape reshape;
2837         int spares_needed;
2838         char *msg;
2839         int orig_level = UnSet;
2840         int odisks;
2841         int delayed;
2842
2843         struct mdu_array_info_s array;
2844         char *c;
2845
2846         struct mddev_dev *dv;
2847         int added_disks;
2848
2849         int *fdlist = NULL;
2850         unsigned long long *offsets = NULL;
2851         int d;
2852         int nrdisks;
2853         int err;
2854         unsigned long blocks;
2855         unsigned long long array_size;
2856         int done;
2857         struct mdinfo *sra = NULL;
2858         char buf[20];
2859
2860         /* when reshaping a RAID0, the component_size might be zero.
2861          * So try to fix that up.
2862          */
2863         if (ioctl(fd, GET_ARRAY_INFO, &array) != 0) {
2864                 dprintf("Cannot get array information.\n");
2865                 goto release;
2866         }
2867         if (array.level == 0 && info->component_size == 0) {
2868                 get_dev_size(fd, NULL, &array_size);
2869                 info->component_size = array_size / array.raid_disks;
2870         }
2871
2872         if (array.level == 10)
2873                 /* Need space_after info */
2874                 get_space_after(fd, st, info);
2875
2876         if (info->reshape_active) {
2877                 int new_level = info->new_level;
2878                 info->new_level = UnSet;
2879                 if (info->delta_disks > 0)
2880                         info->array.raid_disks -= info->delta_disks;
2881                 msg = analyse_change(devname, info, &reshape);
2882                 info->new_level = new_level;
2883                 if (info->delta_disks > 0)
2884                         info->array.raid_disks += info->delta_disks;
2885                 if (!restart)
2886                         /* Make sure the array isn't read-only */
2887                         ioctl(fd, RESTART_ARRAY_RW, 0);
2888         } else
2889                 msg = analyse_change(devname, info, &reshape);
2890         if (msg) {
2891                 /* if msg == "", error has already been printed */
2892                 if (msg[0])
2893                         pr_err("%s\n", msg);
2894                 goto release;
2895         }
2896         if (restart &&
2897             (reshape.level != info->array.level ||
2898              reshape.before.layout != info->array.layout ||
2899              reshape.before.data_disks + reshape.parity
2900              != info->array.raid_disks - max(0, info->delta_disks))) {
2901                 pr_err("reshape info is not in native format - cannot continue.\n");
2902                 goto release;
2903         }
2904
2905         if (st->ss->external && restart && (info->reshape_progress == 0) &&
2906             !((sysfs_get_str(info, NULL, "sync_action", buf, sizeof(buf)) > 0) &&
2907               (strncmp(buf, "reshape", 7) == 0))) {
2908                 /* When reshape is restarted from '0', very begin of array
2909                  * it is possible that for external metadata reshape and array
2910                  * configuration doesn't happen.
2911                  * Check if md has the same opinion, and reshape is restarted
2912                  * from 0. If so, this is regular reshape start after reshape
2913                  * switch in metadata to next array only.
2914                  */
2915                 if ((verify_reshape_position(info, reshape.level) >= 0) &&
2916                     (info->reshape_progress == 0))
2917                         restart = 0;
2918         }
2919         if (restart) {
2920                 /* reshape already started. just skip to monitoring the reshape */
2921                 if (reshape.backup_blocks == 0)
2922                         return 0;
2923                 if (restart & RESHAPE_NO_BACKUP)
2924                         return 0;
2925
2926                 /* Need 'sra' down at 'started:' */
2927                 sra = sysfs_read(fd, NULL,
2928                                  GET_COMPONENT|GET_DEVS|GET_OFFSET|GET_STATE|GET_CHUNK|
2929                                  GET_CACHE);
2930                 if (!sra) {
2931                         pr_err("%s: Cannot get array details from sysfs\n",
2932                                devname);
2933                         goto release;
2934                 }
2935
2936                 if (!backup_file)
2937                         backup_file = locate_backup(sra->sys_name);
2938
2939                 goto started;
2940         }
2941         /* The container is frozen but the array may not be.
2942          * So freeze the array so spares don't get put to the wrong use
2943          * FIXME there should probably be a cleaner separation between
2944          * freeze_array and freeze_container.
2945          */
2946         sysfs_freeze_array(info);
2947         /* Check we have enough spares to not be degraded */
2948         added_disks = 0;
2949         for (dv = devlist; dv ; dv=dv->next)
2950                 added_disks++;
2951         spares_needed = max(reshape.before.data_disks,
2952                             reshape.after.data_disks)
2953                 + reshape.parity - array.raid_disks;
2954
2955         if (!force &&
2956             info->new_level > 1 && info->array.level > 1 &&
2957             spares_needed > info->array.spare_disks + added_disks) {
2958                 pr_err("Need %d spare%s to avoid degraded array, and only have %d.\n"
2959                        "       Use --force to over-ride this check.\n",
2960                        spares_needed,
2961                        spares_needed == 1 ? "" : "s",
2962                        info->array.spare_disks + added_disks);
2963                 goto release;
2964         }
2965         /* Check we have enough spares to not fail */
2966         spares_needed = max(reshape.before.data_disks,
2967                             reshape.after.data_disks)
2968                 - array.raid_disks;
2969         if ((info->new_level > 1 || info->new_level == 0) &&
2970             spares_needed > info->array.spare_disks +added_disks) {
2971                 pr_err("Need %d spare%s to create working array, and only have %d.\n",
2972                        spares_needed,
2973                        spares_needed == 1 ? "" : "s",
2974                        info->array.spare_disks + added_disks);
2975                 goto release;
2976         }
2977
2978         if (reshape.level != array.level) {
2979                 int err = impose_level(fd, reshape.level, devname, verbose);
2980                 if (err)
2981                         goto release;
2982                 info->new_layout = UnSet; /* after level change,
2983                                            * layout is meaningless */
2984                 orig_level = array.level;
2985                 sysfs_freeze_array(info);
2986
2987                 if (reshape.level > 0 && st->ss->external) {
2988                         /* make sure mdmon is aware of the new level */
2989                         if (mdmon_running(container))
2990                                 flush_mdmon(container);
2991
2992                         if (!mdmon_running(container))
2993                                 start_mdmon(container);
2994                         ping_monitor(container);
2995                         if (mdmon_running(container) &&
2996                             st->update_tail == NULL)
2997                                 st->update_tail = &st->updates;
2998                 }
2999         }
3000         /* ->reshape_super might have chosen some spares from the
3001          * container that it wants to be part of the new array.
3002          * We can collect them with ->container_content and give
3003          * them to the kernel.
3004          */
3005         if (st->ss->reshape_super && st->ss->container_content) {
3006                 char *subarray = strchr(info->text_version+1, '/')+1;
3007                 struct mdinfo *info2 =
3008                         st->ss->container_content(st, subarray);
3009                 struct mdinfo *d;
3010
3011                 if (info2) {
3012                         sysfs_init(info2, fd, st->devnm);
3013                         /* When increasing number of devices, we need to set
3014                          * new raid_disks before adding these, or they might
3015                          * be rejected.
3016                          */
3017                         if (reshape.backup_blocks &&
3018                             reshape.after.data_disks > reshape.before.data_disks)
3019                                 subarray_set_num(container, info2, "raid_disks",
3020                                                  reshape.after.data_disks +
3021                                                  reshape.parity);
3022                         for (d = info2->devs; d; d = d->next) {
3023                                 if (d->disk.state == 0 &&
3024                                     d->disk.raid_disk >= 0) {
3025                                         /* This is a spare that wants to
3026                                          * be part of the array.
3027                                          */
3028                                         add_disk(fd, st, info2, d);
3029                                 }
3030                         }
3031                         sysfs_free(info2);
3032                 }
3033         }
3034         /* We might have been given some devices to add to the
3035          * array.  Now that the array has been changed to the right
3036          * level and frozen, we can safely add them.
3037          */
3038         if (devlist) {
3039                 if (Manage_subdevs(devname, fd, devlist, verbose,
3040                                    0, NULL, 0))
3041                         goto release;
3042         }
3043
3044         if (reshape.backup_blocks == 0 && data_offset != INVALID_SECTORS)
3045                 reshape.backup_blocks = reshape.before.data_disks * info->array.chunk_size/512;
3046         if (reshape.backup_blocks == 0) {
3047                 /* No restriping needed, but we might need to impose
3048                  * some more changes: layout, raid_disks, chunk_size
3049                  */
3050                 /* read current array info */
3051                 if (ioctl(fd, GET_ARRAY_INFO, &array) != 0) {
3052                         dprintf("Cannot get array information.\n");
3053                         goto release;
3054                 }
3055                 /* compare current array info with new values and if
3056                  * it is different update them to new */
3057                 if (info->new_layout != UnSet &&
3058                     info->new_layout != array.layout) {
3059                         array.layout = info->new_layout;
3060                         if (ioctl(fd, SET_ARRAY_INFO, &array) != 0) {
3061                                 pr_err("failed to set new layout\n");
3062                                 goto release;
3063                         } else if (verbose >= 0)
3064                                 printf("layout for %s set to %d\n",
3065                                        devname, array.layout);
3066                 }
3067                 if (info->delta_disks != UnSet &&
3068                     info->delta_disks != 0 &&
3069                     array.raid_disks != (info->array.raid_disks + info->delta_disks)) {
3070                         array.raid_disks += info->delta_disks;
3071                         if (ioctl(fd, SET_ARRAY_INFO, &array) != 0) {
3072                                 pr_err("failed to set raid disks\n");
3073                                 goto release;
3074                         } else if (verbose >= 0) {
3075                                 printf("raid_disks for %s set to %d\n",
3076                                        devname, array.raid_disks);
3077                         }
3078                 }
3079                 if (info->new_chunk != 0 &&
3080                     info->new_chunk != array.chunk_size) {
3081                         if (sysfs_set_num(info, NULL,
3082                                           "chunk_size", info->new_chunk) != 0) {
3083                                 pr_err("failed to set chunk size\n");
3084                                 goto release;
3085                         } else if (verbose >= 0)
3086                                 printf("chunk size for %s set to %d\n",
3087                                        devname, array.chunk_size);
3088                 }
3089                 unfreeze(st);
3090                 return 0;
3091         }
3092
3093         /*
3094          * There are three possibilities.
3095          * 1/ The array will shrink.
3096          *    We need to ensure the reshape will pause before reaching
3097          *    the 'critical section'.  We also need to fork and wait for
3098          *    that to happen.  When it does we
3099          *       suspend/backup/complete/unfreeze
3100          *
3101          * 2/ The array will not change size.
3102          *    This requires that we keep a backup of a sliding window
3103          *    so that we can restore data after a crash.  So we need
3104          *    to fork and monitor progress.
3105          *    In future we will allow the data_offset to change, so
3106          *    a sliding backup becomes unnecessary.
3107          *
3108          * 3/ The array will grow. This is relatively easy.
3109          *    However the kernel's restripe routines will cheerfully
3110          *    overwrite some early data before it is safe.  So we
3111          *    need to make a backup of the early parts of the array
3112          *    and be ready to restore it if rebuild aborts very early.
3113          *    For externally managed metadata, we still need a forked
3114          *    child to monitor the reshape and suspend IO over the region
3115          *    that is being reshaped.
3116          *
3117          *    We backup data by writing it to one spare, or to a
3118          *    file which was given on command line.
3119          *
3120          * In each case, we first make sure that storage is available
3121          * for the required backup.
3122          * Then we:
3123          *   -  request the shape change.
3124          *   -  fork to handle backup etc.
3125          */
3126         /* Check that we can hold all the data */
3127         get_dev_size(fd, NULL, &array_size);
3128         if (reshape.new_size < (array_size/512)) {
3129                 pr_err("this change will reduce the size of the array.\n"
3130                        "       use --grow --array-size first to truncate array.\n"
3131                        "       e.g. mdadm --grow %s --array-size %llu\n",
3132                        devname, reshape.new_size/2);
3133                 goto release;
3134         }
3135
3136         if (array.level == 10) {
3137                 /* Reshaping RAID10 does not require any data backup by
3138                  * user-space.  Instead it requires that the data_offset
3139                  * is changed to avoid the need for backup.
3140                  * So this is handled very separately
3141                  */
3142                 if (restart)
3143                         /* Nothing to do. */
3144                         return 0;
3145                 return raid10_reshape(container, fd, devname, st, info,
3146                                       &reshape, data_offset,
3147                                       force, verbose);
3148         }
3149         sra = sysfs_read(fd, NULL,
3150                          GET_COMPONENT|GET_DEVS|GET_OFFSET|GET_STATE|GET_CHUNK|
3151                          GET_CACHE);
3152         if (!sra) {
3153                 pr_err("%s: Cannot get array details from sysfs\n",
3154                         devname);
3155                 goto release;
3156         }
3157
3158         if (!backup_file)
3159                 switch(set_new_data_offset(sra, st, devname,
3160                                            reshape.after.data_disks - reshape.before.data_disks,
3161                                            data_offset,
3162                                            reshape.min_offset_change, 1)) {
3163         case -1:
3164                 goto release;
3165         case 0:
3166                 /* Updated data_offset, so it's easy now */
3167                 update_cache_size(container, sra, info,
3168                                   min(reshape.before.data_disks,
3169                                       reshape.after.data_disks),
3170                                   reshape.backup_blocks);
3171
3172                 /* Right, everything seems fine. Let's kick things off.
3173                  */
3174                 sync_metadata(st);
3175
3176                 if (impose_reshape(sra, info, st, fd, restart,
3177                                    devname, container, &reshape) < 0)
3178                         goto release;
3179                 if (sysfs_set_str(sra, NULL, "sync_action", "reshape") < 0) {
3180                         struct mdinfo *sd;
3181                         if (errno != EINVAL) {
3182                                 pr_err("Failed to initiate reshape!\n");
3183                                 goto release;
3184                         }
3185                         /* revert data_offset and try the old way */
3186                         for (sd = sra->devs; sd; sd = sd->next) {
3187                                 sysfs_set_num(sra, sd, "new_offset",
3188                                               sd->data_offset);
3189                                 sysfs_set_str(sra, NULL, "reshape_direction",
3190                                               "forwards");
3191                         }
3192                         break;
3193                 }
3194                 if (info->new_level == reshape.level)
3195                         return 0;
3196                 /* need to adjust level when reshape completes */
3197                 switch(fork()) {
3198                 case -1: /* ignore error, but don't wait */
3199                         return 0;
3200                 default: /* parent */
3201                         return 0;
3202                 case 0:
3203                         map_fork();
3204                         break;
3205                 }
3206                 close(fd);
3207                 wait_reshape(sra);
3208                 fd = open_dev(sra->sys_name);
3209                 if (fd >= 0)
3210                         impose_level(fd, info->new_level, devname, verbose);
3211                 return 0;
3212         case 1: /* Couldn't set data_offset, try the old way */
3213                 if (data_offset != INVALID_SECTORS) {
3214                         pr_err("Cannot update data_offset on this array\n");
3215                         goto release;
3216                 }
3217                 break;
3218         }
3219
3220 started:
3221         /* Decide how many blocks (sectors) for a reshape
3222          * unit.  The number we have so far is just a minimum
3223          */
3224         blocks = reshape.backup_blocks;
3225         if (reshape.before.data_disks ==
3226             reshape.after.data_disks) {
3227                 /* Make 'blocks' bigger for better throughput, but
3228                  * not so big that we reject it below.
3229                  * Try for 16 megabytes
3230                  */
3231                 while (blocks * 32 < sra->component_size &&
3232                        blocks < 16*1024*2)
3233                         blocks *= 2;
3234         } else
3235                 pr_err("Need to backup %luK of critical section..\n", blocks/2);
3236
3237         if (blocks >= sra->component_size/2) {
3238                 pr_err("%s: Something wrong - reshape aborted\n",
3239                         devname);
3240                 goto release;
3241         }
3242
3243         /* Now we need to open all these devices so we can read/write.
3244          */
3245         nrdisks = max(reshape.before.data_disks,
3246                       reshape.after.data_disks) + reshape.parity
3247                 + sra->array.spare_disks;
3248         fdlist = xcalloc((1+nrdisks), sizeof(int));
3249         offsets = xcalloc((1+nrdisks), sizeof(offsets[0]));
3250
3251         odisks = reshape.before.data_disks + reshape.parity;
3252         d = reshape_prepare_fdlist(devname, sra, odisks,
3253                                    nrdisks, blocks, backup_file,
3254                                    fdlist, offsets);
3255         if (d < odisks) {
3256                 goto release;
3257         }
3258         if ((st->ss->manage_reshape == NULL) ||
3259             (st->ss->recover_backup == NULL)) {
3260                 if (backup_file == NULL) {
3261                         if (reshape.after.data_disks <=
3262                             reshape.before.data_disks) {
3263                                 pr_err("%s: Cannot grow - need backup-file\n",
3264                                        devname);
3265                                 pr_err(" Please provide one with \"--backup=...\"\n");
3266                                 goto release;
3267                         } else if (d == odisks) {
3268                                 pr_err("%s: Cannot grow - need a spare or backup-file to backup critical section\n", devname);
3269                                 goto release;
3270                         }
3271                 } else {
3272                         if (!reshape_open_backup_file(backup_file, fd, devname,
3273                                                       (signed)blocks,
3274                                                       fdlist+d, offsets+d,
3275                                                       sra->sys_name,
3276                                                       restart)) {
3277                                 goto release;
3278                         }
3279                         d++;
3280                 }
3281         }
3282
3283         update_cache_size(container, sra, info,
3284                           min(reshape.before.data_disks, reshape.after.data_disks),
3285                           blocks);
3286
3287         /* Right, everything seems fine. Let's kick things off.
3288          * If only changing raid_disks, use ioctl, else use
3289          * sysfs.
3290          */
3291         sync_metadata(st);
3292
3293         if (impose_reshape(sra, info, st, fd, restart,
3294                            devname, container, &reshape) < 0)
3295                 goto release;
3296
3297         err = start_reshape(sra, restart, reshape.before.data_disks,
3298                             reshape.after.data_disks);
3299         if (err) {
3300                 pr_err("Cannot %s reshape for %s\n",
3301                        restart ? "continue" : "start",
3302                        devname);
3303                 goto release;
3304         }
3305         if (restart)
3306                 sysfs_set_str(sra, NULL, "array_state", "active");
3307         if (freeze_reshape) {
3308                 free(fdlist);
3309                 free(offsets);
3310                 sysfs_free(sra);
3311                 pr_err("Reshape has to be continued from location %llu when root filesystem has been mounted.\n",
3312                         sra->reshape_progress);
3313                 return 1;
3314         }
3315
3316         if (!forked && !check_env("MDADM_NO_SYSTEMCTL"))
3317                 if (continue_via_systemd(container ?: sra->sys_name)) {
3318                         free(fdlist);
3319                         free(offsets);
3320                         sysfs_free(sra);
3321                         return 0;
3322                 }
3323
3324         /* Now we just need to kick off the reshape and watch, while
3325          * handling backups of the data...
3326          * This is all done by a forked background process.
3327          */
3328         switch(forked ? 0 : fork()) {
3329         case -1:
3330                 pr_err("Cannot run child to monitor reshape: %s\n",
3331                         strerror(errno));
3332                 abort_reshape(sra);
3333                 goto release;
3334         default:
3335                 free(fdlist);
3336                 free(offsets);
3337                 sysfs_free(sra);
3338                 return 0;
3339         case 0:
3340                 map_fork();
3341                 break;
3342         }
3343
3344         /* If another array on the same devices is busy, the
3345          * reshape will wait for them.  This would mean that
3346          * the first section that we suspend will stay suspended
3347          * for a long time.  So check on that possibility
3348          * by looking for "DELAYED" in /proc/mdstat, and if found,
3349          * wait a while
3350          */
3351         do {
3352                 struct mdstat_ent *mds, *m;
3353                 delayed = 0;
3354                 mds = mdstat_read(1, 0);
3355                 for (m = mds; m; m = m->next)
3356                         if (strcmp(m->devnm, sra->sys_name) == 0) {
3357                                 if (m->resync &&
3358                                     m->percent == RESYNC_DELAYED)
3359                                         delayed = 1;
3360                                 if (m->resync == 0)
3361                                         /* Haven't started the reshape thread
3362                                          * yet, wait a bit
3363                                          */
3364                                         delayed = 2;
3365                                 break;
3366                         }
3367                 free_mdstat(mds);
3368                 if (delayed == 1 && get_linux_version() < 3007000) {
3369                         pr_err("Reshape is delayed, but cannot wait carefully with this kernel.\n"
3370                                "       You might experience problems until other reshapes complete.\n");
3371                         delayed = 0;
3372                 }
3373                 if (delayed)
3374                         mdstat_wait(30 - (delayed-1) * 25);
3375         } while (delayed);
3376         mdstat_close();
3377         close(fd);
3378         if (check_env("MDADM_GROW_VERIFY"))
3379                 fd = open(devname, O_RDONLY | O_DIRECT);
3380         else
3381                 fd = -1;
3382         mlockall(MCL_FUTURE);
3383
3384         signal(SIGTERM, catch_term);
3385
3386         if (st->ss->external) {
3387                 /* metadata handler takes it from here */
3388                 done = st->ss->manage_reshape(
3389                         fd, sra, &reshape, st, blocks,
3390                         fdlist, offsets,
3391                         d - odisks, fdlist+odisks,
3392                         offsets+odisks);
3393         } else
3394                 done = child_monitor(
3395                         fd, sra, &reshape, st, blocks,
3396                         fdlist, offsets,
3397                         d - odisks, fdlist+odisks,
3398                         offsets+odisks);
3399
3400         free(fdlist);
3401         free(offsets);
3402
3403         if (backup_file && done) {
3404                 char *bul;
3405                 bul = make_backup(sra->sys_name);
3406                 if (bul) {
3407                         char buf[1024];
3408                         int l = readlink(bul, buf, sizeof(buf) - 1);
3409                         if (l > 0) {
3410                                 buf[l]=0;
3411                                 unlink(buf);
3412                         }
3413                         unlink(bul);
3414                         free(bul);
3415                 }
3416                 unlink(backup_file);
3417         }
3418         if (!done) {
3419                 abort_reshape(sra);
3420                 goto out;
3421         }
3422
3423         if (!st->ss->external &&
3424             !(reshape.before.data_disks != reshape.after.data_disks
3425               && info->custom_array_size) &&
3426             info->new_level == reshape.level &&
3427             !forked) {
3428                 /* no need to wait for the reshape to finish as
3429                  * there is nothing more to do.
3430                  */
3431                 sysfs_free(sra);
3432                 exit(0);
3433         }
3434         wait_reshape(sra);
3435
3436         if (st->ss->external) {
3437                 /* Re-load the metadata as much could have changed */
3438                 int cfd = open_dev(st->container_devnm);
3439                 if (cfd >= 0) {
3440                         flush_mdmon(container);
3441                         st->ss->free_super(st);
3442                         st->ss->load_container(st, cfd, container);
3443                         close(cfd);
3444                 }
3445         }
3446
3447         /* set new array size if required customer_array_size is used
3448          * by this metadata.
3449          */
3450         if (reshape.before.data_disks !=
3451             reshape.after.data_disks &&
3452             info->custom_array_size)
3453                 set_array_size(st, info, info->text_version);
3454
3455         if (info->new_level != reshape.level) {
3456                 if (fd < 0)
3457                         fd = open(devname, O_RDONLY);
3458                 impose_level(fd, info->new_level, devname, verbose);
3459                 close(fd);
3460                 if (info->new_level == 0)
3461                         st->update_tail = NULL;
3462         }
3463 out:
3464         sysfs_free(sra);
3465         if (forked)
3466                 return 0;
3467         unfreeze(st);
3468         exit(0);
3469
3470 release:
3471         free(fdlist);
3472         free(offsets);
3473         if (orig_level != UnSet && sra) {
3474                 c = map_num(pers, orig_level);
3475                 if (c && sysfs_set_str(sra, NULL, "level", c) == 0)
3476                         pr_err("aborting level change\n");
3477         }
3478         sysfs_free(sra);
3479         if (!forked)
3480                 unfreeze(st);
3481         return 1;
3482 }
3483
3484 /* mdfd handle is passed to be closed in child process (after fork).
3485  */
3486 int reshape_container(char *container, char *devname,
3487                       int mdfd,
3488                       struct supertype *st,
3489                       struct mdinfo *info,
3490                       int force,
3491                       char *backup_file, int verbose,
3492                       int forked, int restart, int freeze_reshape)
3493 {
3494         struct mdinfo *cc = NULL;
3495         int rv = restart;
3496         char last_devnm[32] = "";
3497
3498         /* component_size is not meaningful for a container,
3499          * so pass '0' meaning 'no change'
3500          */
3501         if (!restart &&
3502             reshape_super(st, 0, info->new_level,
3503                           info->new_layout, info->new_chunk,
3504                           info->array.raid_disks, info->delta_disks,
3505                           backup_file, devname, APPLY_METADATA_CHANGES,
3506                           verbose)) {
3507                 unfreeze(st);
3508                 return 1;
3509         }
3510
3511         sync_metadata(st);
3512
3513         /* ping monitor to be sure that update is on disk
3514          */
3515         ping_monitor(container);
3516
3517         if (!forked && !freeze_reshape && !check_env("MDADM_NO_SYSTEMCTL"))
3518                 if (continue_via_systemd(container))
3519                         return 0;
3520
3521         switch (forked ? 0 : fork()) {
3522         case -1: /* error */
3523                 perror("Cannot fork to complete reshape\n");
3524                 unfreeze(st);
3525                 return 1;
3526         default: /* parent */
3527                 if (!freeze_reshape)
3528                         printf("%s: multi-array reshape continues in background\n", Name);
3529                 return 0;
3530         case 0: /* child */
3531                 map_fork();
3532                 break;
3533         }
3534
3535         /* close unused handle in child process
3536          */
3537         if (mdfd > -1)
3538                 close(mdfd);
3539
3540         while(1) {
3541                 /* For each member array with reshape_active,
3542                  * we need to perform the reshape.
3543                  * We pick the first array that needs reshaping and
3544                  * reshape it.  reshape_array() will re-read the metadata
3545                  * so the next time through a different array should be
3546                  * ready for reshape.
3547                  * It is possible that the 'different' array will not
3548                  * be assembled yet.  In that case we simple exit.
3549                  * When it is assembled, the mdadm which assembles it
3550                  * will take over the reshape.
3551                  */
3552                 struct mdinfo *content;
3553                 int fd;
3554                 struct mdstat_ent *mdstat;
3555                 char *adev;
3556                 dev_t devid;
3557
3558                 sysfs_free(cc);
3559
3560                 cc = st->ss->container_content(st, NULL);
3561
3562                 for (content = cc; content ; content = content->next) {
3563                         char *subarray;
3564                         if (!content->reshape_active)
3565                                 continue;
3566
3567                         subarray = strchr(content->text_version+1, '/')+1;
3568                         mdstat = mdstat_by_subdev(subarray, container);
3569                         if (!mdstat)
3570                                 continue;
3571                         if (mdstat->active == 0) {
3572                                 pr_err("Skipping inactive array %s.\n",
3573                                        mdstat->devnm);
3574                                 free_mdstat(mdstat);
3575                                 mdstat = NULL;
3576                                 continue;
3577                         }
3578                         break;
3579                 }
3580                 if (!content)
3581                         break;
3582
3583                 devid = devnm2devid(mdstat->devnm);
3584                 adev = map_dev(major(devid), minor(devid), 0);
3585                 if (!adev)
3586                         adev = content->text_version;
3587
3588                 fd = open_dev(mdstat->devnm);
3589                 if (fd < 0) {
3590                         pr_err("Device %s cannot be opened for reshape.\n", adev);
3591                         break;
3592                 }
3593
3594                 if (strcmp(last_devnm, mdstat->devnm) == 0) {
3595                         /* Do not allow for multiple reshape_array() calls for
3596                          * the same array.
3597                          * It can happen when reshape_array() returns without
3598                          * error, when reshape is not finished (wrong reshape
3599                          * starting/continuation conditions).  Mdmon doesn't
3600                          * switch to next array in container and reentry
3601                          * conditions for the same array occur.
3602                          * This is possibly interim until the behaviour of
3603                          * reshape_array is resolved().
3604                          */
3605                         printf("%s: Multiple reshape execution detected for device  %s.\n", Name, adev);
3606                         close(fd);
3607                         break;
3608                 }
3609                 strcpy(last_devnm, mdstat->devnm);
3610
3611                 sysfs_init(content, fd, mdstat->devnm);
3612
3613                 if (mdmon_running(container))
3614                         flush_mdmon(container);
3615
3616                 rv = reshape_array(container, fd, adev, st,
3617                                    content, force, NULL, INVALID_SECTORS,
3618                                    backup_file, verbose, 1, restart,
3619                                    freeze_reshape);
3620                 close(fd);
3621
3622                 if (freeze_reshape) {
3623                         sysfs_free(cc);
3624                         exit(0);
3625                 }
3626
3627                 restart = 0;
3628                 if (rv)
3629                         break;
3630
3631                 if (mdmon_running(container))
3632                         flush_mdmon(container);
3633         }
3634         if (!rv)
3635                 unfreeze(st);
3636         sysfs_free(cc);
3637         exit(0);
3638 }
3639
3640 /*
3641  * We run a child process in the background which performs the following
3642  * steps:
3643  *   - wait for resync to reach a certain point
3644  *   - suspend io to the following section
3645  *   - backup that section
3646  *   - allow resync to proceed further
3647  *   - resume io
3648  *   - discard the backup.
3649  *
3650  * When are combined in slightly different ways in the three cases.
3651  * Grow:
3652  *   - suspend/backup/allow/wait/resume/discard
3653  * Shrink:
3654  *   - allow/wait/suspend/backup/allow/wait/resume/discard
3655  * same-size:
3656  *   - wait/resume/discard/suspend/backup/allow
3657  *
3658  * suspend/backup/allow always come together
3659  * wait/resume/discard do too.
3660  * For the same-size case we have two backups to improve flow.
3661  *
3662  */
3663
3664 int progress_reshape(struct mdinfo *info, struct reshape *reshape,
3665                      unsigned long long backup_point,
3666                      unsigned long long wait_point,
3667                      unsigned long long *suspend_point,
3668                      unsigned long long *reshape_completed, int *frozen)
3669 {
3670         /* This function is called repeatedly by the reshape manager.
3671          * It determines how much progress can safely be made and allows
3672          * that progress.
3673          * - 'info' identifies the array and particularly records in
3674          *    ->reshape_progress the metadata's knowledge of progress
3675          *      This is a sector offset from the start of the array
3676          *      of the next array block to be relocated.  This number
3677          *      may increase from 0 or decrease from array_size, depending
3678          *      on the type of reshape that is happening.
3679          *    Note that in contrast, 'sync_completed' is a block count of the
3680          *    reshape so far.  It gives the distance between the start point
3681          *    (head or tail of device) and the next place that data will be
3682          *    written.  It always increases.
3683          * - 'reshape' is the structure created by analyse_change
3684          * - 'backup_point' shows how much the metadata manager has backed-up
3685          *   data.  For reshapes with increasing progress, it is the next address
3686          *   to be backed up, previous addresses have been backed-up.  For
3687          *   decreasing progress, it is the earliest address that has been
3688          *   backed up - later address are also backed up.
3689          *   So addresses between reshape_progress and backup_point are
3690          *   backed up providing those are in the 'correct' order.
3691          * - 'wait_point' is an array address.  When reshape_completed
3692          *   passes this point, progress_reshape should return.  It might
3693          *   return earlier if it determines that ->reshape_progress needs
3694          *   to be updated or further backup is needed.
3695          * - suspend_point is maintained by progress_reshape and the caller
3696          *   should not touch it except to initialise to zero.
3697          *   It is an array address and it only increases in 2.6.37 and earlier.
3698          *   This makes it difficult to handle reducing reshapes with
3699          *   external metadata.
3700          *   However:  it is similar to backup_point in that it records the
3701          *     other end of a suspended region from  reshape_progress.
3702          *     it is moved to extend the region that is safe to backup and/or
3703          *     reshape
3704          * - reshape_completed is read from sysfs and returned.  The caller
3705          *   should copy this into ->reshape_progress when it has reason to
3706          *   believe that the metadata knows this, and any backup outside this
3707          *   has been erased.
3708          *
3709          * Return value is:
3710          *   1 if more data from backup_point - but only as far as suspend_point,
3711          *     should be backed up
3712          *   0 if things are progressing smoothly
3713          *  -1 if the reshape is finished because it is all done,
3714          *  -2 if the reshape is finished due to an error.
3715          */
3716
3717         int advancing = (reshape->after.data_disks
3718                          >= reshape->before.data_disks);
3719         unsigned long long need_backup; /* All data between start of array and
3720                                          * here will at some point need to
3721                                          * be backed up.
3722                                          */
3723         unsigned long long read_offset, write_offset;
3724         unsigned long long write_range;
3725         unsigned long long max_progress, target, completed;
3726         unsigned long long array_size = (info->component_size
3727                                          * reshape->before.data_disks);
3728         int fd;
3729         char buf[20];
3730
3731         /* First, we unsuspend any region that is now known to be safe.
3732          * If suspend_point is on the 'wrong' side of reshape_progress, then
3733          * we don't have or need suspension at the moment.  This is true for
3734          * native metadata when we don't need to back-up.
3735          */
3736         if (advancing) {
3737                 if (info->reshape_progress <= *suspend_point)
3738                         sysfs_set_num(info, NULL, "suspend_lo",
3739                                       info->reshape_progress);
3740         } else {
3741                 /* Note: this won't work in 2.6.37 and before.
3742                  * Something somewhere should make sure we don't need it!
3743                  */
3744                 if (info->reshape_progress >= *suspend_point)
3745                         sysfs_set_num(info, NULL, "suspend_hi",
3746                                       info->reshape_progress);
3747         }
3748
3749         /* Now work out how far it is safe to progress.
3750          * If the read_offset for ->reshape_progress is less than
3751          * 'blocks' beyond the write_offset, we can only progress as far
3752          * as a backup.
3753          * Otherwise we can progress until the write_offset for the new location
3754          * reaches (within 'blocks' of) the read_offset at the current location.
3755          * However that region must be suspended unless we are using native
3756          * metadata.
3757          * If we need to suspend more, we limit it to 128M per device, which is
3758          * rather arbitrary and should be some time-based calculation.
3759          */
3760         read_offset = info->reshape_progress / reshape->before.data_disks;
3761         write_offset = info->reshape_progress / reshape->after.data_disks;
3762         write_range = info->new_chunk/512;
3763         if (reshape->before.data_disks == reshape->after.data_disks)
3764                 need_backup = array_size;
3765         else
3766                 need_backup = reshape->backup_blocks;
3767         if (advancing) {
3768                 if (read_offset < write_offset + write_range)
3769                         max_progress = backup_point;
3770                 else
3771                         max_progress =
3772                                 read_offset *
3773                                 reshape->after.data_disks;
3774         } else {
3775                 if (read_offset > write_offset - write_range)
3776                         /* Can only progress as far as has been backed up,
3777                          * which must be suspended */
3778                         max_progress = backup_point;
3779                 else if (info->reshape_progress <= need_backup)
3780                         max_progress = backup_point;
3781                 else {
3782                         if (info->array.major_version >= 0)
3783                                 /* Can progress until backup is needed */
3784                                 max_progress = need_backup;
3785                         else {
3786                                 /* Can progress until metadata update is required */
3787                                 max_progress =
3788                                         read_offset *
3789                                         reshape->after.data_disks;
3790                                 /* but data must be suspended */
3791                                 if (max_progress < *suspend_point)
3792                                         max_progress = *suspend_point;
3793                         }
3794                 }
3795         }
3796
3797         /* We know it is safe to progress to 'max_progress' providing
3798          * it is suspended or we are using native metadata.
3799          * Consider extending suspend_point 128M per device if it
3800          * is less than 64M per device beyond reshape_progress.
3801          * But always do a multiple of 'blocks'
3802          * FIXME this is too big - it takes to long to complete
3803          * this much.
3804          */
3805         target = 64*1024*2 * min(reshape->before.data_disks,
3806                                  reshape->after.data_disks);
3807         target /= reshape->backup_blocks;
3808         if (target < 2)
3809                 target = 2;
3810         target *= reshape->backup_blocks;
3811
3812         /* For externally managed metadata we always need to suspend IO to
3813          * the area being reshaped so we regularly push suspend_point forward.
3814          * For native metadata we only need the suspend if we are going to do
3815          * a backup.
3816          */
3817         if (advancing) {
3818                 if ((need_backup > info->reshape_progress
3819                      || info->array.major_version < 0) &&
3820                     *suspend_point < info->reshape_progress + target) {
3821                         if (need_backup < *suspend_point + 2 * target)
3822                                 *suspend_point = need_backup;
3823                         else if (*suspend_point + 2 * target < array_size)
3824                                 *suspend_point += 2 * target;
3825                         else
3826                                 *suspend_point = array_size;
3827                         sysfs_set_num(info, NULL, "suspend_hi", *suspend_point);
3828                         if (max_progress > *suspend_point)
3829                                 max_progress = *suspend_point;
3830                 }
3831         } else {
3832                 if (info->array.major_version >= 0) {
3833                         /* Only need to suspend when about to backup */
3834                         if (info->reshape_progress < need_backup * 2 &&
3835                             *suspend_point > 0) {
3836                                 *suspend_point = 0;
3837                                 sysfs_set_num(info, NULL, "suspend_lo", 0);
3838                                 sysfs_set_num(info, NULL, "suspend_hi", need_backup);
3839                         }
3840                 } else {
3841                         /* Need to suspend continually */
3842                         if (info->reshape_progress < *suspend_point)
3843                                 *suspend_point = info->reshape_progress;
3844                         if (*suspend_point + target < info->reshape_progress)
3845                                 /* No need to move suspend region yet */;
3846                         else {
3847                                 if (*suspend_point >= 2 * target)
3848                                         *suspend_point -= 2 * target;
3849                                 else
3850                                         *suspend_point = 0;
3851                                 sysfs_set_num(info, NULL, "suspend_lo",
3852                                               *suspend_point);
3853                         }
3854                         if (max_progress < *suspend_point)
3855                                 max_progress = *suspend_point;
3856                 }
3857         }
3858
3859         /* now set sync_max to allow that progress. sync_max, like
3860          * sync_completed is a count of sectors written per device, so
3861          * we find the difference between max_progress and the start point,
3862          * and divide that by after.data_disks to get a sync_max
3863          * number.
3864          * At the same time we convert wait_point to a similar number
3865          * for comparing against sync_completed.
3866          */
3867         /* scale down max_progress to per_disk */
3868         max_progress /= reshape->after.data_disks;
3869         /* Round to chunk size as some kernels give an erroneously high number */
3870         max_progress /= info->new_chunk/512;
3871         max_progress *= info->new_chunk/512;
3872         /* And round to old chunk size as the kernel wants that */
3873         max_progress /= info->array.chunk_size/512;
3874         max_progress *= info->array.chunk_size/512;
3875         /* Limit progress to the whole device */
3876         if (max_progress > info->component_size)
3877                 max_progress = info->component_size;
3878         wait_point /= reshape->after.data_disks;
3879         if (!advancing) {
3880                 /* switch from 'device offset' to 'processed block count' */
3881                 max_progress = info->component_size - max_progress;
3882                 wait_point = info->component_size - wait_point;
3883         }
3884
3885         if (!*frozen)
3886                 sysfs_set_num(info, NULL, "sync_max", max_progress);
3887
3888         /* Now wait.  If we have already reached the point that we were
3889          * asked to wait to, don't wait at all, else wait for any change.
3890          * We need to select on 'sync_completed' as that is the place that
3891          * notifications happen, but we are really interested in
3892          * 'reshape_position'
3893          */
3894         fd = sysfs_get_fd(info, NULL, "sync_completed");
3895         if (fd < 0)
3896                 goto check_progress;
3897
3898         if (sysfs_fd_get_ll(fd, &completed) < 0)
3899                 goto check_progress;
3900
3901         while (completed < max_progress && completed < wait_point) {
3902                 /* Check that sync_action is still 'reshape' to avoid
3903                  * waiting forever on a dead array
3904                  */
3905                 char action[20];
3906                 if (sysfs_get_str(info, NULL, "sync_action",
3907                                   action, 20) <= 0 ||
3908                     strncmp(action, "reshape", 7) != 0)
3909                         break;
3910                 /* Some kernels reset 'sync_completed' to zero
3911                  * before setting 'sync_action' to 'idle'.
3912                  * So we need these extra tests.
3913                  */
3914                 if (completed == 0 && advancing
3915                     && strncmp(action, "idle", 4) == 0
3916                     && info->reshape_progress > 0)
3917                         break;
3918                 if (completed == 0 && !advancing
3919                     && strncmp(action, "idle", 4) == 0
3920                     && info->reshape_progress < (info->component_size
3921                                                  * reshape->after.data_disks))
3922                         break;
3923                 sysfs_wait(fd, NULL);
3924                 if (sysfs_fd_get_ll(fd, &completed) < 0)
3925                         goto check_progress;
3926         }
3927         /* Some kernels reset 'sync_completed' to zero,
3928          * we need to have real point we are in md.
3929          * So in that case, read 'reshape_position' from sysfs.
3930          */
3931         if (completed == 0) {
3932                 unsigned long long reshapep;
3933                 char action[20];
3934                 if (sysfs_get_str(info, NULL, "sync_action",
3935                                   action, 20) > 0 &&
3936                     strncmp(action, "idle", 4) == 0 &&
3937                     sysfs_get_ll(info, NULL,
3938                                  "reshape_position", &reshapep) == 0)
3939                         *reshape_completed = reshapep;
3940         } else {
3941                 /* some kernels can give an incorrectly high
3942                  * 'completed' number, so round down */
3943                 completed /= (info->new_chunk/512);
3944                 completed *= (info->new_chunk/512);
3945                 /* Convert 'completed' back in to a 'progress' number */
3946                 completed *= reshape->after.data_disks;
3947                 if (!advancing)
3948                         completed = (info->component_size
3949                                      * reshape->after.data_disks
3950                                      - completed);
3951                 *reshape_completed = completed;
3952         }
3953
3954         close(fd);
3955
3956         /* We return the need_backup flag.  Caller will decide
3957          * how much - a multiple of ->backup_blocks up to *suspend_point
3958          */
3959         if (advancing)
3960                 return need_backup > info->reshape_progress;
3961         else
3962                 return need_backup >= info->reshape_progress;
3963
3964 check_progress:
3965         /* if we couldn't read a number from sync_completed, then
3966          * either the reshape did complete, or it aborted.
3967          * We can tell which by checking for 'none' in reshape_position.
3968          * If it did abort, then it might immediately restart if it
3969          * it was just a device failure that leaves us degraded but
3970          * functioning.
3971          */
3972         if (sysfs_get_str(info, NULL, "reshape_position", buf, sizeof(buf)) < 0
3973             || strncmp(buf, "none", 4) != 0) {
3974                 /* The abort might only be temporary.  Wait up to 10
3975                  * seconds for fd to contain a valid number again.
3976                  */
3977                 int wait = 10000;
3978                 int rv = -2;
3979                 unsigned long long new_sync_max;
3980                 while (fd >= 0 && rv < 0 && wait > 0) {
3981                         if (sysfs_wait(fd, &wait) != 1)
3982                                 break;
3983                         switch (sysfs_fd_get_ll(fd, &completed)) {
3984                         case 0:
3985                                 /* all good again */
3986                                 rv = 1;
3987                                 /* If "sync_max" is no longer max_progress
3988                                  * we need to freeze things
3989                                  */
3990                                 sysfs_get_ll(info, NULL, "sync_max", &new_sync_max);
3991                                 *frozen = (new_sync_max != max_progress);
3992                                 break;
3993                         case -2: /* read error - abort */
3994                                 wait = 0;
3995                                 break;
3996                         }
3997                 }
3998                 if (fd >= 0)
3999                         close(fd);
4000                 return rv; /* abort */
4001         } else {
4002                 /* Maybe racing with array shutdown - check state */
4003                 if (fd >= 0)
4004                         close(fd);
4005                 if (sysfs_get_str(info, NULL, "array_state", buf, sizeof(buf)) < 0
4006                     || strncmp(buf, "inactive", 8) == 0
4007                     || strncmp(buf, "clear",5) == 0)
4008                         return -2; /* abort */
4009                 return -1; /* complete */
4010         }
4011 }
4012
4013 /* FIXME return status is never checked */
4014 static int grow_backup(struct mdinfo *sra,
4015                 unsigned long long offset, /* per device */
4016                 unsigned long stripes, /* per device, in old chunks */
4017                 int *sources, unsigned long long *offsets,
4018                 int disks, int chunk, int level, int layout,
4019                 int dests, int *destfd, unsigned long long *destoffsets,
4020                 int part, int *degraded,
4021                 char *buf)
4022 {
4023         /* Backup 'blocks' sectors at 'offset' on each device of the array,
4024          * to storage 'destfd' (offset 'destoffsets'), after first
4025          * suspending IO.  Then allow resync to continue
4026          * over the suspended section.
4027          * Use part 'part' of the backup-super-block.
4028          */
4029         int odata = disks;
4030         int rv = 0;
4031         int i;
4032         unsigned long long ll;
4033         int new_degraded;
4034         //printf("offset %llu\n", offset);
4035         if (level >= 4)
4036                 odata--;
4037         if (level == 6)
4038                 odata--;
4039
4040         /* Check that array hasn't become degraded, else we might backup the wrong data */
4041         if (sysfs_get_ll(sra, NULL, "degraded", &ll) < 0)
4042                 return -1; /* FIXME this error is ignored */
4043         new_degraded = (int)ll;
4044         if (new_degraded != *degraded) {
4045                 /* check each device to ensure it is still working */
4046                 struct mdinfo *sd;
4047                 for (sd = sra->devs ; sd ; sd = sd->next) {
4048                         if (sd->disk.state & (1<<MD_DISK_FAULTY))
4049                                 continue;
4050                         if (sd->disk.state & (1<<MD_DISK_SYNC)) {
4051                                 char sbuf[100];
4052
4053                                 if (sysfs_get_str(sra, sd, "state",
4054                                                   sbuf, sizeof(sbuf)) < 0 ||
4055                                     strstr(sbuf, "faulty") ||
4056                                     strstr(sbuf, "in_sync") == NULL) {
4057                                         /* this device is dead */
4058                                         sd->disk.state = (1<<MD_DISK_FAULTY);
4059                                         if (sd->disk.raid_disk >= 0 &&
4060                                             sources[sd->disk.raid_disk] >= 0) {
4061                                                 close(sources[sd->disk.raid_disk]);
4062                                                 sources[sd->disk.raid_disk] = -1;
4063                                         }
4064                                 }
4065                         }
4066                 }
4067                 *degraded = new_degraded;
4068         }
4069         if (part) {
4070                 bsb.arraystart2 = __cpu_to_le64(offset * odata);
4071                 bsb.length2 = __cpu_to_le64(stripes * (chunk/512) * odata);
4072         } else {
4073                 bsb.arraystart = __cpu_to_le64(offset * odata);
4074                 bsb.length = __cpu_to_le64(stripes * (chunk/512) * odata);
4075         }
4076         if (part)
4077                 bsb.magic[15] = '2';
4078         for (i = 0; i < dests; i++)
4079                 if (part)
4080                         lseek64(destfd[i], destoffsets[i] + __le64_to_cpu(bsb.devstart2)*512, 0);
4081                 else
4082                         lseek64(destfd[i], destoffsets[i], 0);
4083
4084         rv = save_stripes(sources, offsets,
4085                           disks, chunk, level, layout,
4086                           dests, destfd,
4087                           offset*512*odata, stripes * chunk * odata,
4088                           buf);
4089
4090         if (rv)
4091                 return rv;
4092         bsb.mtime = __cpu_to_le64(time(0));
4093         for (i = 0; i < dests; i++) {
4094                 bsb.devstart = __cpu_to_le64(destoffsets[i]/512);
4095
4096                 bsb.sb_csum = bsb_csum((char*)&bsb, ((char*)&bsb.sb_csum)-((char*)&bsb));
4097                 if (memcmp(bsb.magic, "md_backup_data-2", 16) == 0)
4098                         bsb.sb_csum2 = bsb_csum((char*)&bsb,
4099                                                 ((char*)&bsb.sb_csum2)-((char*)&bsb));
4100
4101                 rv = -1;
4102                 if ((unsigned long long)lseek64(destfd[i], destoffsets[i] - 4096, 0)
4103                     != destoffsets[i] - 4096)
4104                         break;
4105                 if (write(destfd[i], &bsb, 512) != 512)
4106                         break;
4107                 if (destoffsets[i] > 4096) {
4108                         if ((unsigned long long)lseek64(destfd[i], destoffsets[i]+stripes*chunk*odata, 0) !=
4109                             destoffsets[i]+stripes*chunk*odata)
4110                                 break;
4111                         if (write(destfd[i], &bsb, 512) != 512)
4112                                 break;
4113                 }
4114                 fsync(destfd[i]);
4115                 rv = 0;
4116         }
4117
4118         return rv;
4119 }
4120
4121 /* in 2.6.30, the value reported by sync_completed can be
4122  * less that it should be by one stripe.
4123  * This only happens when reshape hits sync_max and pauses.
4124  * So allow wait_backup to either extent sync_max further
4125  * than strictly necessary, or return before the
4126  * sync has got quite as far as we would really like.
4127  * This is what 'blocks2' is for.
4128  * The various caller give appropriate values so that
4129  * every works.
4130  */
4131 /* FIXME return value is often ignored */
4132 static int forget_backup(int dests, int *destfd,
4133                          unsigned long long *destoffsets,
4134                          int part)
4135 {
4136         /*
4137          * Erase backup 'part' (which is 0 or 1)
4138          */
4139         int i;
4140         int rv;
4141
4142         if (part) {
4143                 bsb.arraystart2 = __cpu_to_le64(0);
4144                 bsb.length2 = __cpu_to_le64(0);
4145         } else {
4146                 bsb.arraystart = __cpu_to_le64(0);
4147                 bsb.length = __cpu_to_le64(0);
4148         }
4149         bsb.mtime = __cpu_to_le64(time(0));
4150         rv = 0;
4151         for (i = 0; i < dests; i++) {
4152                 bsb.devstart = __cpu_to_le64(destoffsets[i]/512);
4153                 bsb.sb_csum = bsb_csum((char*)&bsb, ((char*)&bsb.sb_csum)-((char*)&bsb));
4154                 if (memcmp(bsb.magic, "md_backup_data-2", 16) == 0)
4155                         bsb.sb_csum2 = bsb_csum((char*)&bsb,
4156                                                 ((char*)&bsb.sb_csum2)-((char*)&bsb));
4157                 if ((unsigned long long)lseek64(destfd[i], destoffsets[i]-4096, 0) !=
4158                     destoffsets[i]-4096)
4159                         rv = -1;
4160                 if (rv == 0 &&
4161                     write(destfd[i], &bsb, 512) != 512)
4162                         rv = -1;
4163                 fsync(destfd[i]);
4164         }
4165         return rv;
4166 }
4167
4168 static void fail(char *msg)
4169 {
4170         int rv;
4171         rv = (write(2, msg, strlen(msg)) != (int)strlen(msg));
4172         rv |= (write(2, "\n", 1) != 1);
4173         exit(rv ? 1 : 2);
4174 }
4175
4176 static char *abuf, *bbuf;
4177 static unsigned long long abuflen;
4178 static void validate(int afd, int bfd, unsigned long long offset)
4179 {
4180         /* check that the data in the backup against the array.
4181          * This is only used for regression testing and should not
4182          * be used while the array is active
4183          */
4184         if (afd < 0)
4185                 return;
4186         lseek64(bfd, offset - 4096, 0);
4187         if (read(bfd, &bsb2, 512) != 512)
4188                 fail("cannot read bsb");
4189         if (bsb2.sb_csum != bsb_csum((char*)&bsb2,
4190                                      ((char*)&bsb2.sb_csum)-((char*)&bsb2)))
4191                 fail("first csum bad");
4192         if (memcmp(bsb2.magic, "md_backup_data", 14) != 0)
4193                 fail("magic is bad");
4194         if (memcmp(bsb2.magic, "md_backup_data-2", 16) == 0 &&
4195             bsb2.sb_csum2 != bsb_csum((char*)&bsb2,
4196                                       ((char*)&bsb2.sb_csum2)-((char*)&bsb2)))
4197                 fail("second csum bad");
4198
4199         if (__le64_to_cpu(bsb2.devstart)*512 != offset)
4200                 fail("devstart is wrong");
4201
4202         if (bsb2.length) {
4203                 unsigned long long len = __le64_to_cpu(bsb2.length)*512;
4204
4205                 if (abuflen < len) {
4206                         free(abuf);
4207                         free(bbuf);
4208                         abuflen = len;
4209                         if (posix_memalign((void**)&abuf, 4096, abuflen) ||
4210                             posix_memalign((void**)&bbuf, 4096, abuflen)) {
4211                                 abuflen = 0;
4212                                 /* just stop validating on mem-alloc failure */
4213                                 return;
4214                         }
4215                 }
4216
4217                 lseek64(bfd, offset, 0);
4218                 if ((unsigned long long)read(bfd, bbuf, len) != len) {
4219                         //printf("len %llu\n", len);
4220                         fail("read first backup failed");
4221                 }
4222                 lseek64(afd, __le64_to_cpu(bsb2.arraystart)*512, 0);
4223                 if ((unsigned long long)read(afd, abuf, len) != len)
4224                         fail("read first from array failed");
4225                 if (memcmp(bbuf, abuf, len) != 0) {
4226 #if 0
4227                         int i;
4228                         printf("offset=%llu len=%llu\n",
4229                                (unsigned long long)__le64_to_cpu(bsb2.arraystart)*512, len);
4230                         for (i=0; i<len; i++)
4231                                 if (bbuf[i] != abuf[i]) {
4232                                         printf("first diff byte %d\n", i);
4233                                         break;
4234                                 }
4235 #endif
4236                         fail("data1 compare failed");
4237                 }
4238         }
4239         if (bsb2.length2) {
4240                 unsigned long long len = __le64_to_cpu(bsb2.length2)*512;
4241
4242                 if (abuflen < len) {
4243                         free(abuf);
4244                         free(bbuf);
4245                         abuflen = len;
4246                         abuf = xmalloc(abuflen);
4247                         bbuf = xmalloc(abuflen);
4248                 }
4249
4250                 lseek64(bfd, offset+__le64_to_cpu(bsb2.devstart2)*512, 0);
4251                 if ((unsigned long long)read(bfd, bbuf, len) != len)
4252                         fail("read second backup failed");
4253                 lseek64(afd, __le64_to_cpu(bsb2.arraystart2)*512, 0);
4254                 if ((unsigned long long)read(afd, abuf, len) != len)
4255                         fail("read second from array failed");
4256                 if (memcmp(bbuf, abuf, len) != 0)
4257                         fail("data2 compare failed");
4258         }
4259 }
4260
4261 int child_monitor(int afd, struct mdinfo *sra, struct reshape *reshape,
4262                   struct supertype *st, unsigned long blocks,
4263                   int *fds, unsigned long long *offsets,
4264                   int dests, int *destfd, unsigned long long *destoffsets)
4265 {
4266         /* Monitor a reshape where backup is being performed using
4267          * 'native' mechanism - either to a backup file, or
4268          * to some space in a spare.
4269          */
4270         char *buf;
4271         int degraded = -1;
4272         unsigned long long speed;
4273         unsigned long long suspend_point, array_size;
4274         unsigned long long backup_point, wait_point;
4275         unsigned long long reshape_completed;
4276         int done = 0;
4277         int increasing = reshape->after.data_disks >= reshape->before.data_disks;
4278         int part = 0; /* The next part of the backup area to fill.  It may already
4279                        * be full, so we need to check */
4280         int level = reshape->level;
4281         int layout = reshape->before.layout;
4282         int data = reshape->before.data_disks;
4283         int disks = reshape->before.data_disks + reshape->parity;
4284         int chunk = sra->array.chunk_size;
4285         struct mdinfo *sd;
4286         unsigned long stripes;
4287         int uuid[4];
4288         int frozen = 0;
4289
4290         /* set up the backup-super-block.  This requires the
4291          * uuid from the array.
4292          */
4293         /* Find a superblock */
4294         for (sd = sra->devs; sd; sd = sd->next) {
4295                 char *dn;
4296                 int devfd;
4297                 int ok;
4298                 if (sd->disk.state & (1<<MD_DISK_FAULTY))
4299                         continue;
4300                 dn = map_dev(sd->disk.major, sd->disk.minor, 1);
4301                 devfd = dev_open(dn, O_RDONLY);
4302                 if (devfd < 0)
4303                         continue;
4304                 ok = st->ss->load_super(st, devfd, NULL);
4305                 close(devfd);
4306                 if (ok == 0)
4307                         break;
4308         }
4309         if (!sd) {
4310                 pr_err("Cannot find a superblock\n");
4311                 return 0;
4312         }
4313
4314         memset(&bsb, 0, 512);
4315         memcpy(bsb.magic, "md_backup_data-1", 16);
4316         st->ss->uuid_from_super(st, uuid);
4317         memcpy(bsb.set_uuid, uuid, 16);
4318         bsb.mtime = __cpu_to_le64(time(0));
4319         bsb.devstart2 = blocks;
4320
4321         stripes = blocks / (sra->array.chunk_size/512) /
4322                 reshape->before.data_disks;
4323
4324         if (posix_memalign((void**)&buf, 4096, disks * chunk))
4325                 /* Don't start the 'reshape' */
4326                 return 0;
4327         if (reshape->before.data_disks == reshape->after.data_disks) {
4328                 sysfs_get_ll(sra, NULL, "sync_speed_min", &speed);
4329                 sysfs_set_num(sra, NULL, "sync_speed_min", 200000);
4330         }
4331
4332         if (increasing) {
4333                 array_size = sra->component_size * reshape->after.data_disks;
4334                 backup_point = sra->reshape_progress;
4335                 suspend_point = 0;
4336         } else {
4337                 array_size = sra->component_size * reshape->before.data_disks;
4338                 backup_point = reshape->backup_blocks;
4339                 suspend_point = array_size;
4340         }
4341
4342         while (!done) {
4343                 int rv;
4344
4345                 /* Want to return as soon the oldest backup slot can
4346                  * be released as that allows us to start backing up
4347                  * some more, providing suspend_point has been
4348                  * advanced, which it should have.
4349                  */
4350                 if (increasing) {
4351                         wait_point = array_size;
4352                         if (part == 0 && __le64_to_cpu(bsb.length) > 0)
4353                                 wait_point = (__le64_to_cpu(bsb.arraystart) +
4354                                               __le64_to_cpu(bsb.length));
4355                         if (part == 1 && __le64_to_cpu(bsb.length2) > 0)
4356                                 wait_point = (__le64_to_cpu(bsb.arraystart2) +
4357                                               __le64_to_cpu(bsb.length2));
4358                 } else {
4359                         wait_point = 0;
4360                         if (part == 0 && __le64_to_cpu(bsb.length) > 0)
4361                                 wait_point = __le64_to_cpu(bsb.arraystart);
4362                         if (part == 1 && __le64_to_cpu(bsb.length2) > 0)
4363                                 wait_point = __le64_to_cpu(bsb.arraystart2);
4364                 }
4365
4366                 reshape_completed = sra->reshape_progress;
4367                 rv = progress_reshape(sra, reshape,
4368                                       backup_point, wait_point,
4369                                       &suspend_point, &reshape_completed,
4370                                       &frozen);
4371                 /* external metadata would need to ping_monitor here */
4372                 sra->reshape_progress = reshape_completed;
4373
4374                 /* Clear any backup region that is before 'here' */
4375                 if (increasing) {
4376                         if (__le64_to_cpu(bsb.length) > 0 &&
4377                             reshape_completed >= (__le64_to_cpu(bsb.arraystart) +
4378                                                   __le64_to_cpu(bsb.length)))
4379                                 forget_backup(dests, destfd,
4380                                               destoffsets, 0);
4381                         if (__le64_to_cpu(bsb.length2) > 0 &&
4382                             reshape_completed >= (__le64_to_cpu(bsb.arraystart2) +
4383                                                   __le64_to_cpu(bsb.length2)))
4384                                 forget_backup(dests, destfd,
4385                                               destoffsets, 1);
4386                 } else {
4387                         if (__le64_to_cpu(bsb.length) > 0 &&
4388                             reshape_completed <= (__le64_to_cpu(bsb.arraystart)))
4389                                 forget_backup(dests, destfd,
4390                                               destoffsets, 0);
4391                         if (__le64_to_cpu(bsb.length2) > 0 &&
4392                             reshape_completed <= (__le64_to_cpu(bsb.arraystart2)))
4393                                 forget_backup(dests, destfd,
4394                                               destoffsets, 1);
4395                 }
4396                 if (sigterm)
4397                         rv = -2;
4398                 if (rv < 0) {
4399                         if (rv == -1)
4400                                 done = 1;
4401                         break;
4402                 }
4403                 if (rv == 0 && increasing && !st->ss->external) {
4404                         /* No longer need to monitor this reshape */
4405                         sysfs_set_str(sra, NULL, "sync_max", "max");
4406                         done = 1;
4407                         break;
4408                 }
4409
4410                 while (rv) {
4411                         unsigned long long offset;
4412                         unsigned long actual_stripes;
4413                         /* Need to backup some data.
4414                          * If 'part' is not used and the desired
4415                          * backup size is suspended, do a backup,
4416                          * then consider the next part.
4417                          */
4418                         /* Check that 'part' is unused */
4419                         if (part == 0 && __le64_to_cpu(bsb.length) != 0)
4420                                 break;
4421                         if (part == 1 && __le64_to_cpu(bsb.length2) != 0)
4422                                 break;
4423
4424                         offset = backup_point / data;
4425                         actual_stripes = stripes;
4426                         if (increasing) {
4427                                 if (offset + actual_stripes * (chunk/512) >
4428                                     sra->component_size)
4429                                         actual_stripes = ((sra->component_size - offset)
4430                                                           / (chunk/512));
4431                                 if (offset + actual_stripes * (chunk/512) >
4432                                     suspend_point/data)
4433                                         break;
4434                         } else {
4435                                 if (offset < actual_stripes * (chunk/512))
4436                                         actual_stripes = offset / (chunk/512);
4437                                 offset -= actual_stripes * (chunk/512);
4438                                 if (offset < suspend_point/data)
4439                                         break;
4440                         }
4441                         if (actual_stripes == 0)
4442                                 break;
4443                         grow_backup(sra, offset, actual_stripes,
4444                                     fds, offsets,
4445                                     disks, chunk, level, layout,
4446                                     dests, destfd, destoffsets,
4447                                     part, &degraded, buf);
4448                         validate(afd, destfd[0], destoffsets[0]);
4449                         /* record where 'part' is up to */
4450                         part = !part;
4451                         if (increasing)
4452                                 backup_point += actual_stripes * (chunk/512) * data;
4453                         else
4454                                 backup_point -= actual_stripes * (chunk/512) * data;
4455                 }
4456         }
4457
4458         /* FIXME maybe call progress_reshape one more time instead */
4459         /* remove any remaining suspension */
4460         sysfs_set_num(sra, NULL, "suspend_lo", 0x7FFFFFFFFFFFFFFFULL);
4461         sysfs_set_num(sra, NULL, "suspend_hi", 0);
4462         sysfs_set_num(sra, NULL, "suspend_lo", 0);
4463         sysfs_set_num(sra, NULL, "sync_min", 0);
4464
4465         if (reshape->before.data_disks == reshape->after.data_disks)
4466                 sysfs_set_num(sra, NULL, "sync_speed_min", speed);
4467         free(buf);
4468         return done;
4469 }
4470
4471 /*
4472  * If any spare contains md_back_data-1 which is recent wrt mtime,
4473  * write that data into the array and update the super blocks with
4474  * the new reshape_progress
4475  */
4476 int Grow_restart(struct supertype *st, struct mdinfo *info, int *fdlist, int cnt,
4477                  char *backup_file, int verbose)
4478 {
4479         int i, j;
4480         int old_disks;
4481         unsigned long long *offsets;
4482         unsigned long long  nstripe, ostripe;
4483         int ndata, odata;
4484
4485         odata = info->array.raid_disks - info->delta_disks - 1;
4486         if (info->array.level == 6) odata--; /* number of data disks */
4487         ndata = info->array.raid_disks - 1;
4488         if (info->new_level == 6) ndata--;
4489
4490         old_disks = info->array.raid_disks - info->delta_disks;
4491
4492         if (info->delta_disks <= 0)
4493                 /* Didn't grow, so the backup file must have
4494                  * been used
4495                  */
4496                 old_disks = cnt;
4497         for (i=old_disks-(backup_file?1:0); i<cnt; i++) {
4498                 struct mdinfo dinfo;
4499                 int fd;
4500                 int bsbsize;
4501                 char *devname, namebuf[20];
4502                 unsigned long long lo, hi;
4503
4504                 /* This was a spare and may have some saved data on it.
4505                  * Load the superblock, find and load the
4506                  * backup_super_block.
4507                  * If either fail, go on to next device.
4508                  * If the backup contains no new info, just return
4509                  * else restore data and update all superblocks
4510                  */
4511                 if (i == old_disks-1) {
4512                         fd = open(backup_file, O_RDONLY);
4513                         if (fd<0) {
4514                                 pr_err("backup file %s inaccessible: %s\n",
4515                                         backup_file, strerror(errno));
4516                                 continue;
4517                         }
4518                         devname = backup_file;
4519                 } else {
4520                         fd = fdlist[i];
4521                         if (fd < 0)
4522                                 continue;
4523                         if (st->ss->load_super(st, fd, NULL))
4524                                 continue;
4525
4526                         st->ss->getinfo_super(st, &dinfo, NULL);
4527                         st->ss->free_super(st);
4528
4529                         if (lseek64(fd,
4530                                     (dinfo.data_offset + dinfo.component_size - 8) <<9,
4531                                     0) < 0) {
4532                                 pr_err("Cannot seek on device %d\n", i);
4533                                 continue; /* Cannot seek */
4534                         }
4535                         sprintf(namebuf, "device-%d", i);
4536                         devname = namebuf;
4537                 }
4538                 if (read(fd, &bsb, sizeof(bsb)) != sizeof(bsb)) {
4539                         if (verbose)
4540                                 pr_err("Cannot read from %s\n", devname);
4541                         continue; /* Cannot read */
4542                 }
4543                 if (memcmp(bsb.magic, "md_backup_data-1", 16) != 0 &&
4544                     memcmp(bsb.magic, "md_backup_data-2", 16) != 0) {
4545                         if (verbose)
4546                                 pr_err("No backup metadata on %s\n", devname);
4547                         continue;
4548                 }
4549                 if (bsb.sb_csum != bsb_csum((char*)&bsb, ((char*)&bsb.sb_csum)-((char*)&bsb))) {
4550                         if (verbose)
4551                                 pr_err("Bad backup-metadata checksum on %s\n", devname);
4552                         continue; /* bad checksum */
4553                 }
4554                 if (memcmp(bsb.magic, "md_backup_data-2", 16) == 0 &&
4555                     bsb.sb_csum2 != bsb_csum((char*)&bsb, ((char*)&bsb.sb_csum2)-((char*)&bsb))) {
4556                         if (verbose)
4557                                 pr_err("Bad backup-metadata checksum2 on %s\n", devname);
4558                         continue; /* Bad second checksum */
4559                 }
4560                 if (memcmp(bsb.set_uuid,info->uuid, 16) != 0) {
4561                         if (verbose)
4562                                 pr_err("Wrong uuid on backup-metadata on %s\n", devname);
4563                         continue; /* Wrong uuid */
4564                 }
4565
4566                 /* array utime and backup-mtime should be updated at much the same time, but it seems that
4567                  * sometimes they aren't... So allow considerable flexability in matching, and allow
4568                  * this test to be overridden by an environment variable.
4569                  */
4570                 if(time_after(info->array.utime, (unsigned int)__le64_to_cpu(bsb.mtime) + 2*60*60) ||
4571                    time_before(info->array.utime, (unsigned int)__le64_to_cpu(bsb.mtime) - 10*60)) {
4572                         if (check_env("MDADM_GROW_ALLOW_OLD")) {
4573                                 pr_err("accepting backup with timestamp %lu for array with timestamp %lu\n",
4574                                         (unsigned long)__le64_to_cpu(bsb.mtime),
4575                                         (unsigned long)info->array.utime);
4576                         } else {
4577                                 pr_err("too-old timestamp on backup-metadata on %s\n", devname);
4578                                 pr_err("If you think it is should be safe, try 'export MDADM_GROW_ALLOW_OLD=1'\n");
4579                                 continue; /* time stamp is too bad */
4580                         }
4581                 }
4582
4583                 if (bsb.magic[15] == '1') {
4584                         if (bsb.length == 0)
4585                                 continue;
4586                         if (info->delta_disks >= 0) {
4587                                 /* reshape_progress is increasing */
4588                                 if (__le64_to_cpu(bsb.arraystart)
4589                                     + __le64_to_cpu(bsb.length)
4590                                     < info->reshape_progress) {
4591                                 nonew:
4592                                         if (verbose)
4593                                                 pr_err("backup-metadata found on %s but is not needed\n", devname);
4594                                         continue; /* No new data here */
4595                                 }
4596                         } else {
4597                                 /* reshape_progress is decreasing */
4598                                 if (__le64_to_cpu(bsb.arraystart) >=
4599                                     info->reshape_progress)
4600                                         goto nonew; /* No new data here */
4601                         }
4602                 } else {
4603                         if (bsb.length == 0 && bsb.length2 == 0)
4604                                 continue;
4605                         if (info->delta_disks >= 0) {
4606                                 /* reshape_progress is increasing */
4607                                 if ((__le64_to_cpu(bsb.arraystart)
4608                                      + __le64_to_cpu(bsb.length)
4609                                      < info->reshape_progress)
4610                                     &&
4611                                     (__le64_to_cpu(bsb.arraystart2)
4612                                      + __le64_to_cpu(bsb.length2)
4613                                      < info->reshape_progress))
4614                                         goto nonew; /* No new data here */
4615                         } else {
4616                                 /* reshape_progress is decreasing */
4617                                 if (__le64_to_cpu(bsb.arraystart) >=
4618                                     info->reshape_progress &&
4619                                     __le64_to_cpu(bsb.arraystart2) >=
4620                                     info->reshape_progress)
4621                                         goto nonew; /* No new data here */
4622                         }
4623                 }
4624                 if (lseek64(fd, __le64_to_cpu(bsb.devstart)*512, 0)< 0) {
4625                 second_fail:
4626                         if (verbose)
4627                                 pr_err("Failed to verify secondary backup-metadata block on %s\n",
4628                                        devname);
4629                         continue; /* Cannot seek */
4630                 }
4631                 /* There should be a duplicate backup superblock 4k before here */
4632                 if (lseek64(fd, -4096, 1) < 0 ||
4633                     read(fd, &bsb2, sizeof(bsb2)) != sizeof(bsb2))
4634                         goto second_fail; /* Cannot find leading superblock */
4635                 if (bsb.magic[15] == '1')
4636                         bsbsize = offsetof(struct mdp_backup_super, pad1);
4637                 else
4638                         bsbsize = offsetof(struct mdp_backup_super, pad);
4639                 if (memcmp(&bsb2, &bsb, bsbsize) != 0)
4640                         goto second_fail; /* Cannot find leading superblock */
4641
4642                 /* Now need the data offsets for all devices. */
4643                 offsets = xmalloc(sizeof(*offsets)*info->array.raid_disks);
4644                 for(j=0; j<info->array.raid_disks; j++) {
4645                         if (fdlist[j] < 0)
4646                                 continue;
4647                         if (st->ss->load_super(st, fdlist[j], NULL))
4648                                 /* FIXME should be this be an error */
4649                                 continue;
4650                         st->ss->getinfo_super(st, &dinfo, NULL);
4651                         st->ss->free_super(st);
4652                         offsets[j] = dinfo.data_offset * 512;
4653                 }
4654                 printf("%s: restoring critical section\n", Name);
4655
4656                 if (restore_stripes(fdlist, offsets,
4657                                     info->array.raid_disks,
4658                                     info->new_chunk,
4659                                     info->new_level,
4660                                     info->new_layout,
4661                                     fd, __le64_to_cpu(bsb.devstart)*512,
4662                                     __le64_to_cpu(bsb.arraystart)*512,
4663                                     __le64_to_cpu(bsb.length)*512, NULL)) {
4664                         /* didn't succeed, so giveup */
4665                         if (verbose)
4666                                 pr_err("Error restoring backup from %s\n",
4667                                         devname);
4668                         free(offsets);
4669                         return 1;
4670                 }
4671
4672                 if (bsb.magic[15] == '2' &&
4673                     restore_stripes(fdlist, offsets,
4674                                     info->array.raid_disks,
4675                                     info->new_chunk,
4676                                     info->new_level,
4677                                     info->new_layout,
4678                                     fd, __le64_to_cpu(bsb.devstart)*512 +
4679                                     __le64_to_cpu(bsb.devstart2)*512,
4680                                     __le64_to_cpu(bsb.arraystart2)*512,
4681                                     __le64_to_cpu(bsb.length2)*512, NULL)) {
4682                         /* didn't succeed, so giveup */
4683                         if (verbose)
4684                                 pr_err("Error restoring second backup from %s\n",
4685                                         devname);
4686                         free(offsets);
4687                         return 1;
4688                 }
4689
4690                 free(offsets);
4691
4692                 /* Ok, so the data is restored. Let's update those superblocks. */
4693
4694                 lo = hi = 0;
4695                 if (bsb.length) {
4696                         lo = __le64_to_cpu(bsb.arraystart);
4697                         hi = lo + __le64_to_cpu(bsb.length);
4698                 }
4699                 if (bsb.magic[15] == '2' && bsb.length2) {
4700                         unsigned long long lo1, hi1;
4701                         lo1 = __le64_to_cpu(bsb.arraystart2);
4702                         hi1 = lo1 + __le64_to_cpu(bsb.length2);
4703                         if (lo == hi) {
4704                                 lo = lo1;
4705                                 hi = hi1;
4706                         } else if (lo < lo1)
4707                                 hi = hi1;
4708                         else
4709                                 lo = lo1;
4710                 }
4711                 if (lo < hi &&
4712                     (info->reshape_progress < lo ||
4713                      info->reshape_progress > hi))
4714                         /* backup does not affect reshape_progress*/ ;
4715                 else if (info->delta_disks >= 0) {
4716                         info->reshape_progress = __le64_to_cpu(bsb.arraystart) +
4717                                 __le64_to_cpu(bsb.length);
4718                         if (bsb.magic[15] == '2') {
4719                                 unsigned long long p2 = __le64_to_cpu(bsb.arraystart2) +
4720                                         __le64_to_cpu(bsb.length2);
4721                                 if (p2 > info->reshape_progress)
4722                                         info->reshape_progress = p2;
4723                         }
4724                 } else {
4725                         info->reshape_progress = __le64_to_cpu(bsb.arraystart);
4726                         if (bsb.magic[15] == '2') {
4727                                 unsigned long long p2 = __le64_to_cpu(bsb.arraystart2);
4728                                 if (p2 < info->reshape_progress)
4729                                         info->reshape_progress = p2;
4730                         }
4731                 }
4732                 for (j=0; j<info->array.raid_disks; j++) {
4733                         if (fdlist[j] < 0)
4734                                 continue;
4735                         if (st->ss->load_super(st, fdlist[j], NULL))
4736                                 continue;
4737                         st->ss->getinfo_super(st, &dinfo, NULL);
4738                         dinfo.reshape_progress = info->reshape_progress;
4739                         st->ss->update_super(st, &dinfo,
4740                                              "_reshape_progress",
4741                                              NULL,0, 0, NULL);
4742                         st->ss->store_super(st, fdlist[j]);
4743                         st->ss->free_super(st);
4744                 }
4745                 return 0;
4746         }
4747         /* Didn't find any backup data, try to see if any
4748          * was needed.
4749          */
4750         if (info->delta_disks < 0) {
4751                 /* When shrinking, the critical section is at the end.
4752                  * So see if we are before the critical section.
4753                  */
4754                 unsigned long long first_block;
4755                 nstripe = ostripe = 0;
4756                 first_block = 0;
4757                 while (ostripe >= nstripe) {
4758                         ostripe += info->array.chunk_size / 512;
4759                         first_block = ostripe * odata;
4760                         nstripe = first_block / ndata / (info->new_chunk/512) *
4761                                 (info->new_chunk/512);
4762                 }
4763
4764                 if (info->reshape_progress >= first_block)
4765                         return 0;
4766         }
4767         if (info->delta_disks > 0) {
4768                 /* See if we are beyond the critical section. */
4769                 unsigned long long last_block;
4770                 nstripe = ostripe = 0;
4771                 last_block = 0;
4772                 while (nstripe >= ostripe) {
4773                         nstripe += info->new_chunk / 512;
4774                         last_block = nstripe * ndata;
4775                         ostripe = last_block / odata / (info->array.chunk_size/512) *
4776                                 (info->array.chunk_size/512);
4777                 }
4778
4779                 if (info->reshape_progress >= last_block)
4780                         return 0;
4781         }
4782         /* needed to recover critical section! */
4783         if (verbose)
4784                 pr_err("Failed to find backup of critical section\n");
4785         return 1;
4786 }
4787
4788 int Grow_continue_command(char *devname, int fd,
4789                           char *backup_file, int verbose)
4790 {
4791         int ret_val = 0;
4792         struct supertype *st = NULL;
4793         struct mdinfo *content = NULL;
4794         struct mdinfo array;
4795         char *subarray = NULL;
4796         struct mdinfo *cc = NULL;
4797         struct mdstat_ent *mdstat = NULL;
4798         int cfd = -1;
4799         int fd2;
4800
4801         dprintf("Grow continue from command line called for %s\n",
4802                 devname);
4803
4804         st = super_by_fd(fd, &subarray);
4805         if (!st || !st->ss) {
4806                 pr_err("Unable to determine metadata format for %s\n",
4807                        devname);
4808                 return 1;
4809         }
4810         dprintf("Grow continue is run for ");
4811         if (st->ss->external == 0) {
4812                 int d;
4813                 int cnt = 5;
4814                 dprintf_cont("native array (%s)\n", devname);
4815                 if (ioctl(fd, GET_ARRAY_INFO, &array.array) < 0) {
4816                         pr_err("%s is not an active md array - aborting\n", devname);
4817                         ret_val = 1;
4818                         goto Grow_continue_command_exit;
4819                 }
4820                 content = &array;
4821                 /* Need to load a superblock.
4822                  * FIXME we should really get what we need from
4823                  * sysfs
4824                  */
4825                 do {
4826                         for (d = 0; d < MAX_DISKS; d++) {
4827                                 mdu_disk_info_t disk;
4828                                 char *dv;
4829                                 int err;
4830                                 disk.number = d;
4831                                 if (ioctl(fd, GET_DISK_INFO, &disk) < 0)
4832                                         continue;
4833                                 if (disk.major == 0 && disk.minor == 0)
4834                                         continue;
4835                                 if ((disk.state & (1 << MD_DISK_ACTIVE)) == 0)
4836                                         continue;
4837                                 dv = map_dev(disk.major, disk.minor, 1);
4838                                 if (!dv)
4839                                         continue;
4840                                 fd2 = dev_open(dv, O_RDONLY);
4841                                 if (fd2 < 0)
4842                                         continue;
4843                                 err = st->ss->load_super(st, fd2, NULL);
4844                                 close(fd2);
4845                                 if (err)
4846                                         continue;
4847                                 break;
4848                         }
4849                         if (d == MAX_DISKS) {
4850                                 pr_err("Unable to load metadata for %s\n",
4851                                        devname);
4852                                 ret_val = 1;
4853                                 goto Grow_continue_command_exit;
4854                         }
4855                         st->ss->getinfo_super(st, content, NULL);
4856                         if (!content->reshape_active)
4857                                 sleep(3);
4858                         else
4859                                 break;
4860                 } while (cnt-- > 0);
4861         } else {
4862                 char *container;
4863
4864                 if (subarray) {
4865                         dprintf_cont("subarray (%s)\n", subarray);
4866                         container = st->container_devnm;
4867                         cfd = open_dev_excl(st->container_devnm);
4868                 } else {
4869                         container = st->devnm;
4870                         close(fd);
4871                         cfd = open_dev_excl(st->devnm);
4872                         dprintf_cont("container (%s)\n", container);
4873                         fd = cfd;
4874                 }
4875                 if (cfd < 0) {
4876                         pr_err("Unable to open container for %s\n", devname);
4877                         ret_val = 1;
4878                         goto Grow_continue_command_exit;
4879                 }
4880
4881                 /* find in container array under reshape
4882                  */
4883                 ret_val = st->ss->load_container(st, cfd, NULL);
4884                 if (ret_val) {
4885                         pr_err("Cannot read superblock for %s\n",
4886                                devname);
4887                         ret_val = 1;
4888                         goto Grow_continue_command_exit;
4889                 }
4890
4891                 cc = st->ss->container_content(st, subarray);
4892                 for (content = cc; content ; content = content->next) {
4893                         char *array;
4894                         int allow_reshape = 1;
4895
4896                         if (content->reshape_active == 0)
4897                                 continue;
4898                         /* The decision about array or container wide
4899                          * reshape is taken in Grow_continue based
4900                          * content->reshape_active state, therefore we
4901                          * need to check_reshape based on
4902                          * reshape_active and subarray name
4903                          */
4904                         if (content->array.state & (1<<MD_SB_BLOCK_VOLUME))
4905                                 allow_reshape = 0;
4906                         if (content->reshape_active == CONTAINER_RESHAPE &&
4907                             (content->array.state
4908                              & (1<<MD_SB_BLOCK_CONTAINER_RESHAPE)))
4909                                 allow_reshape = 0;
4910
4911                         if (!allow_reshape) {
4912                                 pr_err("cannot continue reshape of an array in container with unsupported metadata: %s(%s)\n",
4913                                        devname, container);
4914                                 ret_val = 1;
4915                                 goto Grow_continue_command_exit;
4916                         }
4917
4918                         array = strchr(content->text_version+1, '/')+1;
4919                         mdstat = mdstat_by_subdev(array, container);
4920                         if (!mdstat)
4921                                 continue;
4922                         if (mdstat->active == 0) {
4923                                 pr_err("Skipping inactive array %s.\n",
4924                                        mdstat->devnm);
4925                                 free_mdstat(mdstat);
4926                                 mdstat = NULL;
4927                                 continue;
4928                         }
4929                         break;
4930                 }
4931                 if (!content) {
4932                         pr_err("Unable to determine reshaped array for %s\n", devname);
4933                         ret_val = 1;
4934                         goto Grow_continue_command_exit;
4935                 }
4936                 fd2 = open_dev(mdstat->devnm);
4937                 if (fd2 < 0) {
4938                         pr_err("cannot open (%s)\n", mdstat->devnm);
4939                         ret_val = 1;
4940                         goto Grow_continue_command_exit;
4941                 }
4942
4943                 sysfs_init(content, fd2, mdstat->devnm);
4944
4945                 close(fd2);
4946
4947                 /* start mdmon in case it is not running
4948                  */
4949                 if (!mdmon_running(container))
4950                         start_mdmon(container);
4951                 ping_monitor(container);
4952
4953                 if (mdmon_running(container))
4954                         st->update_tail = &st->updates;
4955                 else {
4956                         pr_err("No mdmon found. Grow cannot continue.\n");
4957                         ret_val = 1;
4958                         goto Grow_continue_command_exit;
4959                 }
4960         }
4961
4962         /* verify that array under reshape is started from
4963          * correct position
4964          */
4965         if (verify_reshape_position(content, content->array.level) < 0) {
4966                 ret_val = 1;
4967                 goto Grow_continue_command_exit;
4968         }
4969
4970         /* continue reshape
4971          */
4972         ret_val = Grow_continue(fd, st, content, backup_file, 1, 0);
4973
4974 Grow_continue_command_exit:
4975         if (cfd > -1)
4976                 close(cfd);
4977         st->ss->free_super(st);
4978         free_mdstat(mdstat);
4979         sysfs_free(cc);
4980         free(subarray);
4981
4982         return ret_val;
4983 }
4984
4985 int Grow_continue(int mdfd, struct supertype *st, struct mdinfo *info,
4986                   char *backup_file, int forked, int freeze_reshape)
4987 {
4988         int ret_val = 2;
4989
4990         if (!info->reshape_active)
4991                 return ret_val;
4992
4993         if (st->ss->external) {
4994                 int cfd = open_dev(st->container_devnm);
4995
4996                 if (cfd < 0)
4997                         return 1;
4998
4999                 st->ss->load_container(st, cfd, st->container_devnm);
5000                 close(cfd);
5001                 ret_val = reshape_container(st->container_devnm, NULL, mdfd,
5002                                             st, info, 0, backup_file,
5003                                             0, forked,
5004                                             1 | info->reshape_active,
5005                                             freeze_reshape);
5006         } else
5007                 ret_val = reshape_array(NULL, mdfd, "array", st, info, 1,
5008                                         NULL, INVALID_SECTORS,
5009                                         backup_file, 0, forked,
5010                                         1 | info->reshape_active,
5011                                         freeze_reshape);
5012
5013         return ret_val;
5014 }
5015
5016 char *make_backup(char *name)
5017 {
5018         char *base = "backup_file-";
5019         int len;
5020         char *fname;
5021
5022         len = strlen(MAP_DIR) + 1 + strlen(base) + strlen(name)+1;
5023         fname = xmalloc(len);
5024         sprintf(fname, "%s/%s%s", MAP_DIR, base, name);
5025         return fname;
5026 }
5027
5028 char *locate_backup(char *name)
5029 {
5030         char *fl = make_backup(name);
5031         struct stat stb;
5032
5033         if (stat(fl, &stb) == 0 &&
5034             S_ISREG(stb.st_mode))
5035                 return fl;
5036
5037         free(fl);
5038         return NULL;
5039 }