blob: ed6a6ecef8d4ca50c795567dc1eb8574dd700b35 [file] [log] [blame]
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001/*
2 * fs/f2fs/segment.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#include <linux/fs.h>
12#include <linux/f2fs_fs.h>
13#include <linux/bio.h>
14#include <linux/blkdev.h>
15#include <linux/prefetch.h>
Evan McClainf3f030d2014-07-17 21:16:35 -040016#include <linux/kthread.h>
Linus Torvalds8005ecc2012-12-20 13:54:51 -080017#include <linux/vmalloc.h>
Changman Leeb1a94e82013-11-15 10:42:51 +090018#include <linux/swap.h>
Linus Torvalds8005ecc2012-12-20 13:54:51 -080019
20#include "f2fs.h"
21#include "segment.h"
22#include "node.h"
23#include <trace/events/f2fs.h>
24
Changman Leeb1a94e82013-11-15 10:42:51 +090025#define __reverse_ffz(x) __reverse_ffs(~(x))
26
27static struct kmem_cache *discard_entry_slab;
28
29/*
30 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
31 * MSB and LSB are reversed in a byte by f2fs_set_bit.
32 */
33static inline unsigned long __reverse_ffs(unsigned long word)
34{
35 int num = 0;
36
37#if BITS_PER_LONG == 64
38 if ((word & 0xffffffff) == 0) {
39 num += 32;
40 word >>= 32;
41 }
42#endif
43 if ((word & 0xffff) == 0) {
44 num += 16;
45 word >>= 16;
46 }
47 if ((word & 0xff) == 0) {
48 num += 8;
49 word >>= 8;
50 }
51 if ((word & 0xf0) == 0)
52 num += 4;
53 else
54 word >>= 4;
55 if ((word & 0xc) == 0)
56 num += 2;
57 else
58 word >>= 2;
59 if ((word & 0x2) == 0)
60 num += 1;
61 return num;
62}
63
64/*
arter97f4081402014-08-06 23:22:50 +090065 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
Changman Leeb1a94e82013-11-15 10:42:51 +090066 * f2fs_set_bit makes MSB and LSB reversed in a byte.
67 * Example:
68 * LSB <--> MSB
69 * f2fs_set_bit(0, bitmap) => 0000 0001
70 * f2fs_set_bit(7, bitmap) => 1000 0000
71 */
72static unsigned long __find_rev_next_bit(const unsigned long *addr,
73 unsigned long size, unsigned long offset)
74{
75 const unsigned long *p = addr + BIT_WORD(offset);
76 unsigned long result = offset & ~(BITS_PER_LONG - 1);
77 unsigned long tmp;
78 unsigned long mask, submask;
79 unsigned long quot, rest;
80
81 if (offset >= size)
82 return size;
83
84 size -= result;
85 offset %= BITS_PER_LONG;
86 if (!offset)
87 goto aligned;
88
89 tmp = *(p++);
90 quot = (offset >> 3) << 3;
91 rest = offset & 0x7;
92 mask = ~0UL << quot;
93 submask = (unsigned char)(0xff << rest) >> rest;
94 submask <<= quot;
95 mask &= submask;
96 tmp &= mask;
97 if (size < BITS_PER_LONG)
98 goto found_first;
99 if (tmp)
100 goto found_middle;
101
102 size -= BITS_PER_LONG;
103 result += BITS_PER_LONG;
104aligned:
105 while (size & ~(BITS_PER_LONG-1)) {
106 tmp = *(p++);
107 if (tmp)
108 goto found_middle;
109 result += BITS_PER_LONG;
110 size -= BITS_PER_LONG;
111 }
112 if (!size)
113 return result;
114 tmp = *p;
115found_first:
116 tmp &= (~0UL >> (BITS_PER_LONG - size));
117 if (tmp == 0UL) /* Are any bits set? */
118 return result + size; /* Nope. */
119found_middle:
120 return result + __reverse_ffs(tmp);
121}
122
123static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
124 unsigned long size, unsigned long offset)
125{
126 const unsigned long *p = addr + BIT_WORD(offset);
127 unsigned long result = offset & ~(BITS_PER_LONG - 1);
128 unsigned long tmp;
129 unsigned long mask, submask;
130 unsigned long quot, rest;
131
132 if (offset >= size)
133 return size;
134
135 size -= result;
136 offset %= BITS_PER_LONG;
137 if (!offset)
138 goto aligned;
139
140 tmp = *(p++);
141 quot = (offset >> 3) << 3;
142 rest = offset & 0x7;
143 mask = ~(~0UL << quot);
144 submask = (unsigned char)~((unsigned char)(0xff << rest) >> rest);
145 submask <<= quot;
146 mask += submask;
147 tmp |= mask;
148 if (size < BITS_PER_LONG)
149 goto found_first;
150 if (~tmp)
151 goto found_middle;
152
153 size -= BITS_PER_LONG;
154 result += BITS_PER_LONG;
155aligned:
156 while (size & ~(BITS_PER_LONG - 1)) {
157 tmp = *(p++);
158 if (~tmp)
159 goto found_middle;
160 result += BITS_PER_LONG;
161 size -= BITS_PER_LONG;
162 }
163 if (!size)
164 return result;
165 tmp = *p;
166
167found_first:
168 tmp |= ~0UL << size;
169 if (tmp == ~0UL) /* Are any bits zero? */
170 return result + size; /* Nope. */
171found_middle:
172 return result + __reverse_ffz(tmp);
173}
174
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800175/*
176 * This function balances dirty node and dentry pages.
177 * In addition, it controls garbage collection.
178 */
179void f2fs_balance_fs(struct f2fs_sb_info *sbi)
180{
181 /*
182 * We should do GC or end up with checkpoint, if there are so many dirty
183 * dir/node pages without enough free segments.
184 */
185 if (has_not_enough_free_secs(sbi, 0)) {
186 mutex_lock(&sbi->gc_mutex);
187 f2fs_gc(sbi);
188 }
189}
190
191void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
192{
193 /* check the # of cached NAT entries and prefree segments */
194 if (try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK) ||
195 excess_prefree_segs(sbi))
196 f2fs_sync_fs(sbi->sb, true);
197}
198
Evan McClainf3f030d2014-07-17 21:16:35 -0400199struct __submit_bio_ret {
200 struct completion event;
201 int error;
202};
203
204static void __submit_bio_wait_endio(struct bio *bio, int error)
205{
206 struct __submit_bio_ret *ret = bio->bi_private;
207
208 ret->error = error;
209 complete(&ret->event);
210}
211
212static int __submit_bio_wait(int rw, struct bio *bio)
213{
214 struct __submit_bio_ret ret;
215
216 rw |= REQ_SYNC;
217 init_completion(&ret.event);
218 bio->bi_private = &ret;
219 bio->bi_end_io = __submit_bio_wait_endio;
220 submit_bio(rw, bio);
221 wait_for_completion(&ret.event);
222
223 return ret.error;
224}
225
226static int issue_flush_thread(void *data)
227{
228 struct f2fs_sb_info *sbi = data;
229 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
230 wait_queue_head_t *q = &fcc->flush_wait_queue;
231repeat:
232 if (kthread_should_stop())
233 return 0;
234
235 spin_lock(&fcc->issue_lock);
236 if (fcc->issue_list) {
237 fcc->dispatch_list = fcc->issue_list;
238 fcc->issue_list = fcc->issue_tail = NULL;
239 }
240 spin_unlock(&fcc->issue_lock);
241
242 if (fcc->dispatch_list) {
243 struct bio *bio = bio_alloc(GFP_NOIO, 0);
244 struct flush_cmd *cmd, *next;
245 int ret;
246
247 bio->bi_bdev = sbi->sb->s_bdev;
248 ret = __submit_bio_wait(WRITE_FLUSH, bio);
249
250 for (cmd = fcc->dispatch_list; cmd; cmd = next) {
251 cmd->ret = ret;
252 next = cmd->next;
253 complete(&cmd->wait);
254 }
255 bio_put(bio);
256 fcc->dispatch_list = NULL;
257 }
258
259 wait_event_interruptible(*q,
260 kthread_should_stop() || fcc->issue_list);
261 goto repeat;
262}
263
264int f2fs_issue_flush(struct f2fs_sb_info *sbi)
265{
266 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
267 struct flush_cmd cmd;
268
Jaegeuk Kim90421d22014-07-25 17:46:10 -0700269 trace_f2fs_issue_flush(sbi->sb, test_opt(sbi, NOBARRIER),
270 test_opt(sbi, FLUSH_MERGE));
271
Jaegeuk Kim6f6541b2014-07-23 09:57:31 -0700272 if (test_opt(sbi, NOBARRIER))
273 return 0;
274
Evan McClainf3f030d2014-07-17 21:16:35 -0400275 if (!test_opt(sbi, FLUSH_MERGE))
276 return blkdev_issue_flush(sbi->sb->s_bdev, GFP_KERNEL, NULL);
277
278 init_completion(&cmd.wait);
279 cmd.next = NULL;
280
281 spin_lock(&fcc->issue_lock);
282 if (fcc->issue_list)
283 fcc->issue_tail->next = &cmd;
284 else
285 fcc->issue_list = &cmd;
286 fcc->issue_tail = &cmd;
287 spin_unlock(&fcc->issue_lock);
288
289 if (!fcc->dispatch_list)
290 wake_up(&fcc->flush_wait_queue);
291
292 wait_for_completion(&cmd.wait);
293
294 return cmd.ret;
295}
296
297int create_flush_cmd_control(struct f2fs_sb_info *sbi)
298{
299 dev_t dev = sbi->sb->s_bdev->bd_dev;
300 struct flush_cmd_control *fcc;
301 int err = 0;
302
303 fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL);
304 if (!fcc)
305 return -ENOMEM;
306 spin_lock_init(&fcc->issue_lock);
307 init_waitqueue_head(&fcc->flush_wait_queue);
308 SM_I(sbi)->cmd_control_info = fcc;
309 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
310 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
311 if (IS_ERR(fcc->f2fs_issue_flush)) {
312 err = PTR_ERR(fcc->f2fs_issue_flush);
313 kfree(fcc);
314 SM_I(sbi)->cmd_control_info = NULL;
315 return err;
316 }
317
318 return err;
319}
320
321void destroy_flush_cmd_control(struct f2fs_sb_info *sbi)
322{
323 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
324
325 if (fcc && fcc->f2fs_issue_flush)
326 kthread_stop(fcc->f2fs_issue_flush);
327 kfree(fcc);
328 SM_I(sbi)->cmd_control_info = NULL;
329}
330
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800331static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
332 enum dirty_type dirty_type)
333{
334 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
335
336 /* need not be added */
337 if (IS_CURSEG(sbi, segno))
338 return;
339
340 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
341 dirty_i->nr_dirty[dirty_type]++;
342
343 if (dirty_type == DIRTY) {
344 struct seg_entry *sentry = get_seg_entry(sbi, segno);
345 enum dirty_type t = sentry->type;
346
347 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
348 dirty_i->nr_dirty[t]++;
349 }
350}
351
352static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
353 enum dirty_type dirty_type)
354{
355 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
356
357 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
358 dirty_i->nr_dirty[dirty_type]--;
359
360 if (dirty_type == DIRTY) {
361 struct seg_entry *sentry = get_seg_entry(sbi, segno);
362 enum dirty_type t = sentry->type;
363
364 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
365 dirty_i->nr_dirty[t]--;
366
367 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0)
368 clear_bit(GET_SECNO(sbi, segno),
369 dirty_i->victim_secmap);
370 }
371}
372
373/*
374 * Should not occur error such as -ENOMEM.
375 * Adding dirty entry into seglist is not critical operation.
376 * If a given segment is one of current working segments, it won't be added.
377 */
378static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
379{
380 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
381 unsigned short valid_blocks;
382
383 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
384 return;
385
386 mutex_lock(&dirty_i->seglist_lock);
387
388 valid_blocks = get_valid_blocks(sbi, segno, 0);
389
390 if (valid_blocks == 0) {
391 __locate_dirty_segment(sbi, segno, PRE);
392 __remove_dirty_segment(sbi, segno, DIRTY);
393 } else if (valid_blocks < sbi->blocks_per_seg) {
394 __locate_dirty_segment(sbi, segno, DIRTY);
395 } else {
396 /* Recovery routine with SSR needs this */
397 __remove_dirty_segment(sbi, segno, DIRTY);
398 }
399
400 mutex_unlock(&dirty_i->seglist_lock);
401}
402
Jaegeuk Kimf8ff1412014-04-15 13:57:55 +0900403static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
Changman Leeb1a94e82013-11-15 10:42:51 +0900404 block_t blkstart, block_t blklen)
405{
406 sector_t start = SECTOR_FROM_BLOCK(sbi, blkstart);
407 sector_t len = SECTOR_FROM_BLOCK(sbi, blklen);
Changman Leeb1a94e82013-11-15 10:42:51 +0900408 trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
Jaegeuk Kimf8ff1412014-04-15 13:57:55 +0900409 return blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0);
410}
411
Dan Pasanen97e8a342014-08-25 19:11:31 -0500412void discard_next_dnode(struct f2fs_sb_info *sbi)
Jaegeuk Kimf8ff1412014-04-15 13:57:55 +0900413{
Dan Pasanen97e8a342014-08-25 19:11:31 -0500414 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_WARM_NODE);
415 block_t blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
416
Jaegeuk Kimf8ff1412014-04-15 13:57:55 +0900417 if (f2fs_issue_discard(sbi, blkaddr, 1)) {
418 struct page *page = grab_meta_page(sbi, blkaddr);
419 /* zero-filled page */
420 set_page_dirty(page);
421 f2fs_put_page(page, 1);
422 }
Changman Leeb1a94e82013-11-15 10:42:51 +0900423}
424
425static void add_discard_addrs(struct f2fs_sb_info *sbi,
426 unsigned int segno, struct seg_entry *se)
427{
428 struct list_head *head = &SM_I(sbi)->discard_list;
429 struct discard_entry *new;
430 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
431 int max_blocks = sbi->blocks_per_seg;
432 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
433 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
434 unsigned long dmap[entries];
435 unsigned int start = 0, end = -1;
436 int i;
437
438 if (!test_opt(sbi, DISCARD))
439 return;
440
441 /* zero block will be discarded through the prefree list */
442 if (!se->valid_blocks || se->valid_blocks == max_blocks)
443 return;
444
445 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
446 for (i = 0; i < entries; i++)
447 dmap[i] = (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
448
449 while (SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) {
450 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
451 if (start >= max_blocks)
452 break;
453
454 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
455
456 new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS);
457 INIT_LIST_HEAD(&new->list);
458 new->blkaddr = START_BLOCK(sbi, segno) + start;
459 new->len = end - start;
460
461 list_add_tail(&new->list, head);
462 SM_I(sbi)->nr_discards += end - start;
463 }
464}
465
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800466/*
467 * Should call clear_prefree_segments after checkpoint is done.
468 */
469static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
470{
471 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
Chao Yu00ebaff2014-08-04 10:10:07 +0800472 unsigned int segno;
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800473 unsigned int total_segs = TOTAL_SEGS(sbi);
474
475 mutex_lock(&dirty_i->seglist_lock);
Chao Yu00ebaff2014-08-04 10:10:07 +0800476 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], total_segs)
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800477 __set_test_and_free(sbi, segno);
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800478 mutex_unlock(&dirty_i->seglist_lock);
479}
480
481void clear_prefree_segments(struct f2fs_sb_info *sbi)
482{
Changman Leeb1a94e82013-11-15 10:42:51 +0900483 struct list_head *head = &(SM_I(sbi)->discard_list);
Chao Yu48c561a2014-03-29 11:33:17 +0800484 struct discard_entry *entry, *this;
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800485 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
486 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
487 unsigned int total_segs = TOTAL_SEGS(sbi);
488 unsigned int start = 0, end = -1;
489
490 mutex_lock(&dirty_i->seglist_lock);
491
492 while (1) {
493 int i;
494 start = find_next_bit(prefree_map, total_segs, end + 1);
495 if (start >= total_segs)
496 break;
497 end = find_next_zero_bit(prefree_map, total_segs, start + 1);
498
499 for (i = start; i < end; i++)
500 clear_bit(i, prefree_map);
501
502 dirty_i->nr_dirty[PRE] -= end - start;
503
504 if (!test_opt(sbi, DISCARD))
505 continue;
506
Changman Leeb1a94e82013-11-15 10:42:51 +0900507 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
508 (end - start) << sbi->log_blocks_per_seg);
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800509 }
510 mutex_unlock(&dirty_i->seglist_lock);
Changman Leeb1a94e82013-11-15 10:42:51 +0900511
512 /* send small discards */
Chao Yu48c561a2014-03-29 11:33:17 +0800513 list_for_each_entry_safe(entry, this, head, list) {
Changman Leeb1a94e82013-11-15 10:42:51 +0900514 f2fs_issue_discard(sbi, entry->blkaddr, entry->len);
515 list_del(&entry->list);
516 SM_I(sbi)->nr_discards -= entry->len;
517 kmem_cache_free(discard_entry_slab, entry);
518 }
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800519}
520
521static void __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
522{
523 struct sit_info *sit_i = SIT_I(sbi);
524 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap))
525 sit_i->dirty_sentries++;
526}
527
528static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
529 unsigned int segno, int modified)
530{
531 struct seg_entry *se = get_seg_entry(sbi, segno);
532 se->type = type;
533 if (modified)
534 __mark_sit_entry_dirty(sbi, segno);
535}
536
537static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
538{
539 struct seg_entry *se;
540 unsigned int segno, offset;
541 long int new_vblocks;
542
543 segno = GET_SEGNO(sbi, blkaddr);
544
545 se = get_seg_entry(sbi, segno);
546 new_vblocks = se->valid_blocks + del;
Jaegeuk Kim26d12822014-02-04 13:01:10 +0900547 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800548
549 f2fs_bug_on((new_vblocks >> (sizeof(unsigned short) << 3) ||
550 (new_vblocks > sbi->blocks_per_seg)));
551
552 se->valid_blocks = new_vblocks;
553 se->mtime = get_mtime(sbi);
554 SIT_I(sbi)->max_mtime = se->mtime;
555
556 /* Update valid block bitmap */
557 if (del > 0) {
558 if (f2fs_set_bit(offset, se->cur_valid_map))
559 BUG();
560 } else {
561 if (!f2fs_clear_bit(offset, se->cur_valid_map))
562 BUG();
563 }
564 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
565 se->ckpt_valid_blocks += del;
566
567 __mark_sit_entry_dirty(sbi, segno);
568
569 /* update total number of valid blocks to be written in ckpt area */
570 SIT_I(sbi)->written_valid_blocks += del;
571
572 if (sbi->segs_per_sec > 1)
573 get_sec_entry(sbi, segno)->valid_blocks += del;
574}
575
Jaegeuk Kim655d2c12014-01-28 12:22:14 +0900576void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new)
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800577{
Jaegeuk Kim655d2c12014-01-28 12:22:14 +0900578 update_sit_entry(sbi, new, 1);
579 if (GET_SEGNO(sbi, old) != NULL_SEGNO)
580 update_sit_entry(sbi, old, -1);
581
582 locate_dirty_segment(sbi, GET_SEGNO(sbi, old));
583 locate_dirty_segment(sbi, GET_SEGNO(sbi, new));
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800584}
585
586void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
587{
588 unsigned int segno = GET_SEGNO(sbi, addr);
589 struct sit_info *sit_i = SIT_I(sbi);
590
591 f2fs_bug_on(addr == NULL_ADDR);
592 if (addr == NEW_ADDR)
593 return;
594
595 /* add it into sit main buffer */
596 mutex_lock(&sit_i->sentry_lock);
597
598 update_sit_entry(sbi, addr, -1);
599
600 /* add it into dirty seglist */
601 locate_dirty_segment(sbi, segno);
602
603 mutex_unlock(&sit_i->sentry_lock);
604}
605
606/*
607 * This function should be resided under the curseg_mutex lock
608 */
609static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
610 struct f2fs_summary *sum)
611{
612 struct curseg_info *curseg = CURSEG_I(sbi, type);
613 void *addr = curseg->sum_blk;
614 addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
615 memcpy(addr, sum, sizeof(struct f2fs_summary));
616}
617
618/*
619 * Calculate the number of current summary pages for writing
620 */
621int npages_for_summary_flush(struct f2fs_sb_info *sbi)
622{
623 int valid_sum_count = 0;
624 int i, sum_in_page;
625
626 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
627 if (sbi->ckpt->alloc_type[i] == SSR)
628 valid_sum_count += sbi->blocks_per_seg;
629 else
630 valid_sum_count += curseg_blkoff(sbi, i);
631 }
632
633 sum_in_page = (PAGE_CACHE_SIZE - 2 * SUM_JOURNAL_SIZE -
634 SUM_FOOTER_SIZE) / SUMMARY_SIZE;
635 if (valid_sum_count <= sum_in_page)
636 return 1;
637 else if ((valid_sum_count - sum_in_page) <=
638 (PAGE_CACHE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
639 return 2;
640 return 3;
641}
642
643/*
644 * Caller should put this summary page
645 */
646struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
647{
648 return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno));
649}
650
651static void write_sum_page(struct f2fs_sb_info *sbi,
652 struct f2fs_summary_block *sum_blk, block_t blk_addr)
653{
654 struct page *page = grab_meta_page(sbi, blk_addr);
655 void *kaddr = page_address(page);
656 memcpy(kaddr, sum_blk, PAGE_CACHE_SIZE);
657 set_page_dirty(page);
658 f2fs_put_page(page, 1);
659}
660
661static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
662{
663 struct curseg_info *curseg = CURSEG_I(sbi, type);
664 unsigned int segno = curseg->segno + 1;
665 struct free_segmap_info *free_i = FREE_I(sbi);
666
667 if (segno < TOTAL_SEGS(sbi) && segno % sbi->segs_per_sec)
668 return !test_bit(segno, free_i->free_segmap);
669 return 0;
670}
671
672/*
673 * Find a new segment from the free segments bitmap to right order
674 * This function should be returned with success, otherwise BUG
675 */
676static void get_new_segment(struct f2fs_sb_info *sbi,
677 unsigned int *newseg, bool new_sec, int dir)
678{
679 struct free_segmap_info *free_i = FREE_I(sbi);
680 unsigned int segno, secno, zoneno;
681 unsigned int total_zones = TOTAL_SECS(sbi) / sbi->secs_per_zone;
682 unsigned int hint = *newseg / sbi->segs_per_sec;
683 unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg);
684 unsigned int left_start = hint;
685 bool init = true;
686 int go_left = 0;
687 int i;
688
689 write_lock(&free_i->segmap_lock);
690
691 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
692 segno = find_next_zero_bit(free_i->free_segmap,
693 TOTAL_SEGS(sbi), *newseg + 1);
694 if (segno - *newseg < sbi->segs_per_sec -
695 (*newseg % sbi->segs_per_sec))
696 goto got_it;
697 }
698find_other_zone:
699 secno = find_next_zero_bit(free_i->free_secmap, TOTAL_SECS(sbi), hint);
700 if (secno >= TOTAL_SECS(sbi)) {
701 if (dir == ALLOC_RIGHT) {
702 secno = find_next_zero_bit(free_i->free_secmap,
703 TOTAL_SECS(sbi), 0);
704 f2fs_bug_on(secno >= TOTAL_SECS(sbi));
705 } else {
706 go_left = 1;
707 left_start = hint - 1;
708 }
709 }
710 if (go_left == 0)
711 goto skip_left;
712
713 while (test_bit(left_start, free_i->free_secmap)) {
714 if (left_start > 0) {
715 left_start--;
716 continue;
717 }
718 left_start = find_next_zero_bit(free_i->free_secmap,
719 TOTAL_SECS(sbi), 0);
720 f2fs_bug_on(left_start >= TOTAL_SECS(sbi));
721 break;
722 }
723 secno = left_start;
724skip_left:
725 hint = secno;
726 segno = secno * sbi->segs_per_sec;
727 zoneno = secno / sbi->secs_per_zone;
728
729 /* give up on finding another zone */
730 if (!init)
731 goto got_it;
732 if (sbi->secs_per_zone == 1)
733 goto got_it;
734 if (zoneno == old_zoneno)
735 goto got_it;
736 if (dir == ALLOC_LEFT) {
737 if (!go_left && zoneno + 1 >= total_zones)
738 goto got_it;
739 if (go_left && zoneno == 0)
740 goto got_it;
741 }
742 for (i = 0; i < NR_CURSEG_TYPE; i++)
743 if (CURSEG_I(sbi, i)->zone == zoneno)
744 break;
745
746 if (i < NR_CURSEG_TYPE) {
747 /* zone is in user, try another */
748 if (go_left)
749 hint = zoneno * sbi->secs_per_zone - 1;
750 else if (zoneno + 1 >= total_zones)
751 hint = 0;
752 else
753 hint = (zoneno + 1) * sbi->secs_per_zone;
754 init = false;
755 goto find_other_zone;
756 }
757got_it:
758 /* set it as dirty segment in free segmap */
759 f2fs_bug_on(test_bit(segno, free_i->free_segmap));
760 __set_inuse(sbi, segno);
761 *newseg = segno;
762 write_unlock(&free_i->segmap_lock);
763}
764
765static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
766{
767 struct curseg_info *curseg = CURSEG_I(sbi, type);
768 struct summary_footer *sum_footer;
769
770 curseg->segno = curseg->next_segno;
771 curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno);
772 curseg->next_blkoff = 0;
773 curseg->next_segno = NULL_SEGNO;
774
775 sum_footer = &(curseg->sum_blk->footer);
776 memset(sum_footer, 0, sizeof(struct summary_footer));
777 if (IS_DATASEG(type))
778 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
779 if (IS_NODESEG(type))
780 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
781 __set_sit_entry_type(sbi, type, curseg->segno, modified);
782}
783
784/*
785 * Allocate a current working segment.
786 * This function always allocates a free segment in LFS manner.
787 */
788static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
789{
790 struct curseg_info *curseg = CURSEG_I(sbi, type);
791 unsigned int segno = curseg->segno;
792 int dir = ALLOC_LEFT;
793
794 write_sum_page(sbi, curseg->sum_blk,
795 GET_SUM_BLOCK(sbi, segno));
796 if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA)
797 dir = ALLOC_RIGHT;
798
799 if (test_opt(sbi, NOHEAP))
800 dir = ALLOC_RIGHT;
801
802 get_new_segment(sbi, &segno, new_sec, dir);
803 curseg->next_segno = segno;
804 reset_curseg(sbi, type, 1);
805 curseg->alloc_type = LFS;
806}
807
808static void __next_free_blkoff(struct f2fs_sb_info *sbi,
809 struct curseg_info *seg, block_t start)
810{
811 struct seg_entry *se = get_seg_entry(sbi, seg->segno);
Changman Leeb1a94e82013-11-15 10:42:51 +0900812 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
813 unsigned long target_map[entries];
814 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
815 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
816 int i, pos;
817
818 for (i = 0; i < entries; i++)
819 target_map[i] = ckpt_map[i] | cur_map[i];
820
821 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
822
823 seg->next_blkoff = pos;
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800824}
825
826/*
827 * If a segment is written by LFS manner, next block offset is just obtained
828 * by increasing the current block offset. However, if a segment is written by
829 * SSR manner, next block offset obtained by calling __next_free_blkoff
830 */
831static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
832 struct curseg_info *seg)
833{
834 if (seg->alloc_type == SSR)
835 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
836 else
837 seg->next_blkoff++;
838}
839
840/*
arter97f4081402014-08-06 23:22:50 +0900841 * This function always allocates a used segment(from dirty seglist) by SSR
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800842 * manner, so it should recover the existing segment information of valid blocks
843 */
844static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)
845{
846 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
847 struct curseg_info *curseg = CURSEG_I(sbi, type);
848 unsigned int new_segno = curseg->next_segno;
849 struct f2fs_summary_block *sum_node;
850 struct page *sum_page;
851
852 write_sum_page(sbi, curseg->sum_blk,
853 GET_SUM_BLOCK(sbi, curseg->segno));
854 __set_test_and_inuse(sbi, new_segno);
855
856 mutex_lock(&dirty_i->seglist_lock);
857 __remove_dirty_segment(sbi, new_segno, PRE);
858 __remove_dirty_segment(sbi, new_segno, DIRTY);
859 mutex_unlock(&dirty_i->seglist_lock);
860
861 reset_curseg(sbi, type, 1);
862 curseg->alloc_type = SSR;
863 __next_free_blkoff(sbi, curseg, 0);
864
865 if (reuse) {
866 sum_page = get_sum_page(sbi, new_segno);
867 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
868 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
869 f2fs_put_page(sum_page, 1);
870 }
871}
872
873static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
874{
875 struct curseg_info *curseg = CURSEG_I(sbi, type);
876 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
877
878 if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0))
879 return v_ops->get_victim(sbi,
880 &(curseg)->next_segno, BG_GC, type, SSR);
881
882 /* For data segments, let's do SSR more intensively */
883 for (; type >= CURSEG_HOT_DATA; type--)
884 if (v_ops->get_victim(sbi, &(curseg)->next_segno,
885 BG_GC, type, SSR))
886 return 1;
887 return 0;
888}
889
890/*
891 * flush out current segment and replace it with new segment
892 * This function should be returned with success, otherwise BUG
893 */
894static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
895 int type, bool force)
896{
897 struct curseg_info *curseg = CURSEG_I(sbi, type);
898
899 if (force)
900 new_curseg(sbi, type, true);
901 else if (type == CURSEG_WARM_NODE)
902 new_curseg(sbi, type, false);
903 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))
904 new_curseg(sbi, type, false);
905 else if (need_SSR(sbi) && get_ssr_segment(sbi, type))
906 change_curseg(sbi, type, true);
907 else
908 new_curseg(sbi, type, false);
909
910 stat_inc_seg_type(sbi, curseg);
911}
912
913void allocate_new_segments(struct f2fs_sb_info *sbi)
914{
915 struct curseg_info *curseg;
916 unsigned int old_curseg;
917 int i;
918
919 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
920 curseg = CURSEG_I(sbi, i);
921 old_curseg = curseg->segno;
922 SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true);
923 locate_dirty_segment(sbi, old_curseg);
924 }
925}
926
927static const struct segment_allocation default_salloc_ops = {
928 .allocate_segment = allocate_segment_by_default,
929};
930
Linus Torvalds8005ecc2012-12-20 13:54:51 -0800931static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
932{
933 struct curseg_info *curseg = CURSEG_I(sbi, type);
934 if (curseg->next_blkoff < sbi->blocks_per_seg)
935 return true;
936 return false;
937}
938
939static int __get_segment_type_2(struct page *page, enum page_type p_type)
940{
941 if (p_type == DATA)
942 return CURSEG_HOT_DATA;
943 else
944 return CURSEG_HOT_NODE;
945}
946
947static int __get_segment_type_4(struct page *page, enum page_type p_type)
948{
949 if (p_type == DATA) {
950 struct inode *inode = page->mapping->host;
951
952 if (S_ISDIR(inode->i_mode))
953 return CURSEG_HOT_DATA;
954 else
955 return CURSEG_COLD_DATA;
956 } else {
957 if (IS_DNODE(page) && !is_cold_node(page))
958 return CURSEG_HOT_NODE;
959 else
960 return CURSEG_COLD_NODE;
961 }
962}
963
964static int __get_segment_type_6(struct page *page, enum page_type p_type)
965{
966 if (p_type == DATA) {
967 struct inode *inode = page->mapping->host;
968
969 if (S_ISDIR(inode->i_mode))
970 return CURSEG_HOT_DATA;
971 else if (is_cold_data(page) || file_is_cold(inode))
972 return CURSEG_COLD_DATA;
973 else
974 return CURSEG_WARM_DATA;
975 } else {
976 if (IS_DNODE(page))
977 return is_cold_node(page) ? CURSEG_WARM_NODE :
978 CURSEG_HOT_NODE;
979 else
980 return CURSEG_COLD_NODE;
981 }
982}
983
984static int __get_segment_type(struct page *page, enum page_type p_type)
985{
986 struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
987 switch (sbi->active_logs) {
988 case 2:
989 return __get_segment_type_2(page, p_type);
990 case 4:
991 return __get_segment_type_4(page, p_type);
992 }
993 /* NR_CURSEG_TYPE(6) logs by default */
994 f2fs_bug_on(sbi->active_logs != NR_CURSEG_TYPE);
995 return __get_segment_type_6(page, p_type);
996}
997
Changman Leeb1a94e82013-11-15 10:42:51 +0900998void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
999 block_t old_blkaddr, block_t *new_blkaddr,
1000 struct f2fs_summary *sum, int type)
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001001{
1002 struct sit_info *sit_i = SIT_I(sbi);
1003 struct curseg_info *curseg;
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001004
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001005 curseg = CURSEG_I(sbi, type);
1006
1007 mutex_lock(&curseg->curseg_mutex);
1008
1009 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001010
1011 /*
1012 * __add_sum_entry should be resided under the curseg_mutex
1013 * because, this function updates a summary entry in the
1014 * current summary block.
1015 */
1016 __add_sum_entry(sbi, type, sum);
1017
1018 mutex_lock(&sit_i->sentry_lock);
1019 __refresh_next_blkoff(sbi, curseg);
1020
1021 stat_inc_block_count(sbi, curseg);
1022
Jaegeuk Kim655d2c12014-01-28 12:22:14 +09001023 if (!__has_curseg_space(sbi, type))
1024 sit_i->s_ops->allocate_segment(sbi, type, false);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001025 /*
1026 * SIT information should be updated before segment allocation,
1027 * since SSR needs latest valid block information.
1028 */
1029 refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr);
Jaegeuk Kim655d2c12014-01-28 12:22:14 +09001030
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001031 mutex_unlock(&sit_i->sentry_lock);
1032
Changman Leeb1a94e82013-11-15 10:42:51 +09001033 if (page && IS_NODESEG(type))
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001034 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
1035
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001036 mutex_unlock(&curseg->curseg_mutex);
1037}
1038
Changman Leeb1a94e82013-11-15 10:42:51 +09001039static void do_write_page(struct f2fs_sb_info *sbi, struct page *page,
1040 block_t old_blkaddr, block_t *new_blkaddr,
1041 struct f2fs_summary *sum, struct f2fs_io_info *fio)
1042{
1043 int type = __get_segment_type(page, fio->type);
1044
1045 allocate_data_block(sbi, page, old_blkaddr, new_blkaddr, sum, type);
1046
1047 /* writeout dirty page into bdev */
1048 f2fs_submit_page_mbio(sbi, page, *new_blkaddr, fio);
1049}
1050
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001051void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
1052{
Changman Leeb1a94e82013-11-15 10:42:51 +09001053 struct f2fs_io_info fio = {
1054 .type = META,
1055 .rw = WRITE_SYNC | REQ_META | REQ_PRIO
1056 };
1057
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001058 set_page_writeback(page);
Changman Leeb1a94e82013-11-15 10:42:51 +09001059 f2fs_submit_page_mbio(sbi, page, page->index, &fio);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001060}
1061
1062void write_node_page(struct f2fs_sb_info *sbi, struct page *page,
Changman Leeb1a94e82013-11-15 10:42:51 +09001063 struct f2fs_io_info *fio,
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001064 unsigned int nid, block_t old_blkaddr, block_t *new_blkaddr)
1065{
1066 struct f2fs_summary sum;
1067 set_summary(&sum, nid, 0, 0);
Changman Leeb1a94e82013-11-15 10:42:51 +09001068 do_write_page(sbi, page, old_blkaddr, new_blkaddr, &sum, fio);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001069}
1070
Changman Leeb1a94e82013-11-15 10:42:51 +09001071void write_data_page(struct page *page, struct dnode_of_data *dn,
1072 block_t *new_blkaddr, struct f2fs_io_info *fio)
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001073{
Changman Leeb1a94e82013-11-15 10:42:51 +09001074 struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001075 struct f2fs_summary sum;
1076 struct node_info ni;
1077
Changman Leeb1a94e82013-11-15 10:42:51 +09001078 f2fs_bug_on(dn->data_blkaddr == NULL_ADDR);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001079 get_node_info(sbi, dn->nid, &ni);
1080 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1081
Changman Leeb1a94e82013-11-15 10:42:51 +09001082 do_write_page(sbi, page, dn->data_blkaddr, new_blkaddr, &sum, fio);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001083}
1084
Changman Leeb1a94e82013-11-15 10:42:51 +09001085void rewrite_data_page(struct page *page, block_t old_blkaddr,
1086 struct f2fs_io_info *fio)
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001087{
Changman Leeb1a94e82013-11-15 10:42:51 +09001088 struct inode *inode = page->mapping->host;
1089 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
1090 f2fs_submit_page_mbio(sbi, page, old_blkaddr, fio);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001091}
1092
1093void recover_data_page(struct f2fs_sb_info *sbi,
1094 struct page *page, struct f2fs_summary *sum,
1095 block_t old_blkaddr, block_t new_blkaddr)
1096{
1097 struct sit_info *sit_i = SIT_I(sbi);
1098 struct curseg_info *curseg;
1099 unsigned int segno, old_cursegno;
1100 struct seg_entry *se;
1101 int type;
1102
1103 segno = GET_SEGNO(sbi, new_blkaddr);
1104 se = get_seg_entry(sbi, segno);
1105 type = se->type;
1106
1107 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
1108 if (old_blkaddr == NULL_ADDR)
1109 type = CURSEG_COLD_DATA;
1110 else
1111 type = CURSEG_WARM_DATA;
1112 }
1113 curseg = CURSEG_I(sbi, type);
1114
1115 mutex_lock(&curseg->curseg_mutex);
1116 mutex_lock(&sit_i->sentry_lock);
1117
1118 old_cursegno = curseg->segno;
1119
1120 /* change the current segment */
1121 if (segno != curseg->segno) {
1122 curseg->next_segno = segno;
1123 change_curseg(sbi, type, true);
1124 }
1125
Jaegeuk Kim26d12822014-02-04 13:01:10 +09001126 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001127 __add_sum_entry(sbi, type, sum);
1128
1129 refresh_sit_entry(sbi, old_blkaddr, new_blkaddr);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001130 locate_dirty_segment(sbi, old_cursegno);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001131
1132 mutex_unlock(&sit_i->sentry_lock);
1133 mutex_unlock(&curseg->curseg_mutex);
1134}
1135
Evan McClainf3f030d2014-07-17 21:16:35 -04001136static inline bool is_merged_page(struct f2fs_sb_info *sbi,
1137 struct page *page, enum page_type type)
1138{
1139 enum page_type btype = PAGE_TYPE_OF_BIO(type);
1140 struct f2fs_bio_info *io = &sbi->write_io[btype];
1141 struct bio_vec *bvec;
1142 int i;
1143
1144 down_read(&io->io_rwsem);
1145 if (!io->bio)
1146 goto out;
1147
1148 __bio_for_each_segment(bvec, io->bio, i, 0) {
1149 if (page == bvec->bv_page) {
1150 up_read(&io->io_rwsem);
1151 return true;
1152 }
1153 }
1154
1155out:
1156 up_read(&io->io_rwsem);
1157 return false;
1158}
1159
Changman Leeb1a94e82013-11-15 10:42:51 +09001160void f2fs_wait_on_page_writeback(struct page *page,
1161 enum page_type type)
1162{
1163 struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
1164 if (PageWriteback(page)) {
Evan McClainf3f030d2014-07-17 21:16:35 -04001165 if (is_merged_page(sbi, page, type))
1166 f2fs_submit_merged_bio(sbi, type, WRITE);
Changman Leeb1a94e82013-11-15 10:42:51 +09001167 wait_on_page_writeback(page);
1168 }
1169}
1170
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001171static int read_compacted_summaries(struct f2fs_sb_info *sbi)
1172{
1173 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1174 struct curseg_info *seg_i;
1175 unsigned char *kaddr;
1176 struct page *page;
1177 block_t start;
1178 int i, j, offset;
1179
1180 start = start_sum_block(sbi);
1181
1182 page = get_meta_page(sbi, start++);
1183 kaddr = (unsigned char *)page_address(page);
1184
1185 /* Step 1: restore nat cache */
1186 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1187 memcpy(&seg_i->sum_blk->n_nats, kaddr, SUM_JOURNAL_SIZE);
1188
1189 /* Step 2: restore sit cache */
1190 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1191 memcpy(&seg_i->sum_blk->n_sits, kaddr + SUM_JOURNAL_SIZE,
1192 SUM_JOURNAL_SIZE);
1193 offset = 2 * SUM_JOURNAL_SIZE;
1194
1195 /* Step 3: restore summary entries */
1196 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1197 unsigned short blk_off;
1198 unsigned int segno;
1199
1200 seg_i = CURSEG_I(sbi, i);
1201 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
1202 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
1203 seg_i->next_segno = segno;
1204 reset_curseg(sbi, i, 0);
1205 seg_i->alloc_type = ckpt->alloc_type[i];
1206 seg_i->next_blkoff = blk_off;
1207
1208 if (seg_i->alloc_type == SSR)
1209 blk_off = sbi->blocks_per_seg;
1210
1211 for (j = 0; j < blk_off; j++) {
1212 struct f2fs_summary *s;
1213 s = (struct f2fs_summary *)(kaddr + offset);
1214 seg_i->sum_blk->entries[j] = *s;
1215 offset += SUMMARY_SIZE;
1216 if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1217 SUM_FOOTER_SIZE)
1218 continue;
1219
1220 f2fs_put_page(page, 1);
1221 page = NULL;
1222
1223 page = get_meta_page(sbi, start++);
1224 kaddr = (unsigned char *)page_address(page);
1225 offset = 0;
1226 }
1227 }
1228 f2fs_put_page(page, 1);
1229 return 0;
1230}
1231
1232static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
1233{
1234 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1235 struct f2fs_summary_block *sum;
1236 struct curseg_info *curseg;
1237 struct page *new;
1238 unsigned short blk_off;
1239 unsigned int segno = 0;
1240 block_t blk_addr = 0;
1241
1242 /* get segment number and block addr */
1243 if (IS_DATASEG(type)) {
1244 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
1245 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
1246 CURSEG_HOT_DATA]);
1247 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG))
1248 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
1249 else
1250 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
1251 } else {
1252 segno = le32_to_cpu(ckpt->cur_node_segno[type -
1253 CURSEG_HOT_NODE]);
1254 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
1255 CURSEG_HOT_NODE]);
1256 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG))
1257 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
1258 type - CURSEG_HOT_NODE);
1259 else
1260 blk_addr = GET_SUM_BLOCK(sbi, segno);
1261 }
1262
1263 new = get_meta_page(sbi, blk_addr);
1264 sum = (struct f2fs_summary_block *)page_address(new);
1265
1266 if (IS_NODESEG(type)) {
1267 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) {
1268 struct f2fs_summary *ns = &sum->entries[0];
1269 int i;
1270 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
1271 ns->version = 0;
1272 ns->ofs_in_node = 0;
1273 }
1274 } else {
Gu Zheng12e374b2014-03-07 18:43:36 +08001275 int err;
1276
1277 err = restore_node_summary(sbi, segno, sum);
1278 if (err) {
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001279 f2fs_put_page(new, 1);
Gu Zheng12e374b2014-03-07 18:43:36 +08001280 return err;
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001281 }
1282 }
1283 }
1284
1285 /* set uncompleted segment to curseg */
1286 curseg = CURSEG_I(sbi, type);
1287 mutex_lock(&curseg->curseg_mutex);
1288 memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE);
1289 curseg->next_segno = segno;
1290 reset_curseg(sbi, type, 0);
1291 curseg->alloc_type = ckpt->alloc_type[type];
1292 curseg->next_blkoff = blk_off;
1293 mutex_unlock(&curseg->curseg_mutex);
1294 f2fs_put_page(new, 1);
1295 return 0;
1296}
1297
1298static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
1299{
1300 int type = CURSEG_HOT_DATA;
Chao Yu32c234e2014-03-17 16:36:24 +08001301 int err;
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001302
1303 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) {
1304 /* restore for compacted data summary */
1305 if (read_compacted_summaries(sbi))
1306 return -EINVAL;
1307 type = CURSEG_HOT_NODE;
1308 }
1309
Chao Yu32c234e2014-03-17 16:36:24 +08001310 for (; type <= CURSEG_COLD_NODE; type++) {
1311 err = read_normal_summaries(sbi, type);
1312 if (err)
1313 return err;
1314 }
1315
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001316 return 0;
1317}
1318
1319static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
1320{
1321 struct page *page;
1322 unsigned char *kaddr;
1323 struct f2fs_summary *summary;
1324 struct curseg_info *seg_i;
1325 int written_size = 0;
1326 int i, j;
1327
1328 page = grab_meta_page(sbi, blkaddr++);
1329 kaddr = (unsigned char *)page_address(page);
1330
1331 /* Step 1: write nat cache */
1332 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1333 memcpy(kaddr, &seg_i->sum_blk->n_nats, SUM_JOURNAL_SIZE);
1334 written_size += SUM_JOURNAL_SIZE;
1335
1336 /* Step 2: write sit cache */
1337 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1338 memcpy(kaddr + written_size, &seg_i->sum_blk->n_sits,
1339 SUM_JOURNAL_SIZE);
1340 written_size += SUM_JOURNAL_SIZE;
1341
1342 /* Step 3: write summary entries */
1343 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1344 unsigned short blkoff;
1345 seg_i = CURSEG_I(sbi, i);
1346 if (sbi->ckpt->alloc_type[i] == SSR)
1347 blkoff = sbi->blocks_per_seg;
1348 else
1349 blkoff = curseg_blkoff(sbi, i);
1350
1351 for (j = 0; j < blkoff; j++) {
1352 if (!page) {
1353 page = grab_meta_page(sbi, blkaddr++);
1354 kaddr = (unsigned char *)page_address(page);
1355 written_size = 0;
1356 }
1357 summary = (struct f2fs_summary *)(kaddr + written_size);
1358 *summary = seg_i->sum_blk->entries[j];
1359 written_size += SUMMARY_SIZE;
1360
1361 if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1362 SUM_FOOTER_SIZE)
1363 continue;
1364
1365 set_page_dirty(page);
1366 f2fs_put_page(page, 1);
1367 page = NULL;
1368 }
1369 }
1370 if (page) {
1371 set_page_dirty(page);
1372 f2fs_put_page(page, 1);
1373 }
1374}
1375
1376static void write_normal_summaries(struct f2fs_sb_info *sbi,
1377 block_t blkaddr, int type)
1378{
1379 int i, end;
1380 if (IS_DATASEG(type))
1381 end = type + NR_CURSEG_DATA_TYPE;
1382 else
1383 end = type + NR_CURSEG_NODE_TYPE;
1384
1385 for (i = type; i < end; i++) {
1386 struct curseg_info *sum = CURSEG_I(sbi, i);
1387 mutex_lock(&sum->curseg_mutex);
1388 write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type));
1389 mutex_unlock(&sum->curseg_mutex);
1390 }
1391}
1392
1393void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1394{
1395 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG))
1396 write_compacted_summaries(sbi, start_blk);
1397 else
1398 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
1399}
1400
1401void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1402{
1403 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_UMOUNT_FLAG))
1404 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
1405}
1406
1407int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type,
1408 unsigned int val, int alloc)
1409{
1410 int i;
1411
1412 if (type == NAT_JOURNAL) {
1413 for (i = 0; i < nats_in_cursum(sum); i++) {
1414 if (le32_to_cpu(nid_in_journal(sum, i)) == val)
1415 return i;
1416 }
1417 if (alloc && nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES)
1418 return update_nats_in_cursum(sum, 1);
1419 } else if (type == SIT_JOURNAL) {
1420 for (i = 0; i < sits_in_cursum(sum); i++)
1421 if (le32_to_cpu(segno_in_journal(sum, i)) == val)
1422 return i;
1423 if (alloc && sits_in_cursum(sum) < SIT_JOURNAL_ENTRIES)
1424 return update_sits_in_cursum(sum, 1);
1425 }
1426 return -1;
1427}
1428
1429static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
1430 unsigned int segno)
1431{
1432 struct sit_info *sit_i = SIT_I(sbi);
1433 unsigned int offset = SIT_BLOCK_OFFSET(sit_i, segno);
1434 block_t blk_addr = sit_i->sit_base_addr + offset;
1435
1436 check_seg_range(sbi, segno);
1437
1438 /* calculate sit block address */
1439 if (f2fs_test_bit(offset, sit_i->sit_bitmap))
1440 blk_addr += sit_i->sit_blocks;
1441
1442 return get_meta_page(sbi, blk_addr);
1443}
1444
1445static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
1446 unsigned int start)
1447{
1448 struct sit_info *sit_i = SIT_I(sbi);
1449 struct page *src_page, *dst_page;
1450 pgoff_t src_off, dst_off;
1451 void *src_addr, *dst_addr;
1452
1453 src_off = current_sit_addr(sbi, start);
1454 dst_off = next_sit_addr(sbi, src_off);
1455
1456 /* get current sit block page without lock */
1457 src_page = get_meta_page(sbi, src_off);
1458 dst_page = grab_meta_page(sbi, dst_off);
1459 f2fs_bug_on(PageDirty(src_page));
1460
1461 src_addr = page_address(src_page);
1462 dst_addr = page_address(dst_page);
1463 memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE);
1464
1465 set_page_dirty(dst_page);
1466 f2fs_put_page(src_page, 1);
1467
1468 set_to_next_sit(sit_i, start);
1469
1470 return dst_page;
1471}
1472
1473static bool flush_sits_in_journal(struct f2fs_sb_info *sbi)
1474{
1475 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1476 struct f2fs_summary_block *sum = curseg->sum_blk;
1477 int i;
1478
1479 /*
1480 * If the journal area in the current summary is full of sit entries,
1481 * all the sit entries will be flushed. Otherwise the sit entries
1482 * are not able to replace with newly hot sit entries.
1483 */
1484 if (sits_in_cursum(sum) >= SIT_JOURNAL_ENTRIES) {
1485 for (i = sits_in_cursum(sum) - 1; i >= 0; i--) {
1486 unsigned int segno;
1487 segno = le32_to_cpu(segno_in_journal(sum, i));
1488 __mark_sit_entry_dirty(sbi, segno);
1489 }
1490 update_sits_in_cursum(sum, -sits_in_cursum(sum));
1491 return true;
1492 }
1493 return false;
1494}
1495
1496/*
1497 * CP calls this function, which flushes SIT entries including sit_journal,
1498 * and moves prefree segs to free segs.
1499 */
1500void flush_sit_entries(struct f2fs_sb_info *sbi)
1501{
1502 struct sit_info *sit_i = SIT_I(sbi);
1503 unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
1504 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1505 struct f2fs_summary_block *sum = curseg->sum_blk;
1506 unsigned long nsegs = TOTAL_SEGS(sbi);
1507 struct page *page = NULL;
1508 struct f2fs_sit_block *raw_sit = NULL;
1509 unsigned int start = 0, end = 0;
Chao Yu00ebaff2014-08-04 10:10:07 +08001510 unsigned int segno;
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001511 bool flushed;
1512
1513 mutex_lock(&curseg->curseg_mutex);
1514 mutex_lock(&sit_i->sentry_lock);
1515
1516 /*
1517 * "flushed" indicates whether sit entries in journal are flushed
1518 * to the SIT area or not.
1519 */
1520 flushed = flush_sits_in_journal(sbi);
1521
Chao Yu00ebaff2014-08-04 10:10:07 +08001522 for_each_set_bit(segno, bitmap, nsegs) {
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001523 struct seg_entry *se = get_seg_entry(sbi, segno);
1524 int sit_offset, offset;
1525
1526 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
1527
Changman Leeb1a94e82013-11-15 10:42:51 +09001528 /* add discard candidates */
1529 if (SM_I(sbi)->nr_discards < SM_I(sbi)->max_discards)
1530 add_discard_addrs(sbi, segno, se);
1531
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001532 if (flushed)
1533 goto to_sit_page;
1534
1535 offset = lookup_journal_in_cursum(sum, SIT_JOURNAL, segno, 1);
1536 if (offset >= 0) {
1537 segno_in_journal(sum, offset) = cpu_to_le32(segno);
1538 seg_info_to_raw_sit(se, &sit_in_journal(sum, offset));
1539 goto flush_done;
1540 }
1541to_sit_page:
1542 if (!page || (start > segno) || (segno > end)) {
1543 if (page) {
1544 f2fs_put_page(page, 1);
1545 page = NULL;
1546 }
1547
1548 start = START_SEGNO(sit_i, segno);
1549 end = start + SIT_ENTRY_PER_BLOCK - 1;
1550
1551 /* read sit block that will be updated */
1552 page = get_next_sit_page(sbi, start);
1553 raw_sit = page_address(page);
1554 }
1555
1556 /* udpate entry in SIT block */
1557 seg_info_to_raw_sit(se, &raw_sit->entries[sit_offset]);
1558flush_done:
1559 __clear_bit(segno, bitmap);
1560 sit_i->dirty_sentries--;
1561 }
1562 mutex_unlock(&sit_i->sentry_lock);
1563 mutex_unlock(&curseg->curseg_mutex);
1564
1565 /* writeout last modified SIT block */
1566 f2fs_put_page(page, 1);
1567
1568 set_prefree_as_free_segments(sbi);
1569}
1570
1571static int build_sit_info(struct f2fs_sb_info *sbi)
1572{
1573 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
1574 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1575 struct sit_info *sit_i;
1576 unsigned int sit_segs, start;
1577 char *src_bitmap, *dst_bitmap;
1578 unsigned int bitmap_size;
1579
1580 /* allocate memory for SIT information */
1581 sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL);
1582 if (!sit_i)
1583 return -ENOMEM;
1584
1585 SM_I(sbi)->sit_info = sit_i;
1586
1587 sit_i->sentries = vzalloc(TOTAL_SEGS(sbi) * sizeof(struct seg_entry));
1588 if (!sit_i->sentries)
1589 return -ENOMEM;
1590
1591 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));
1592 sit_i->dirty_sentries_bitmap = kzalloc(bitmap_size, GFP_KERNEL);
1593 if (!sit_i->dirty_sentries_bitmap)
1594 return -ENOMEM;
1595
1596 for (start = 0; start < TOTAL_SEGS(sbi); start++) {
1597 sit_i->sentries[start].cur_valid_map
1598 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
1599 sit_i->sentries[start].ckpt_valid_map
1600 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
1601 if (!sit_i->sentries[start].cur_valid_map
1602 || !sit_i->sentries[start].ckpt_valid_map)
1603 return -ENOMEM;
1604 }
1605
1606 if (sbi->segs_per_sec > 1) {
1607 sit_i->sec_entries = vzalloc(TOTAL_SECS(sbi) *
1608 sizeof(struct sec_entry));
1609 if (!sit_i->sec_entries)
1610 return -ENOMEM;
1611 }
1612
1613 /* get information related with SIT */
1614 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
1615
1616 /* setup SIT bitmap from ckeckpoint pack */
1617 bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
1618 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
1619
1620 dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
1621 if (!dst_bitmap)
1622 return -ENOMEM;
1623
1624 /* init SIT information */
1625 sit_i->s_ops = &default_salloc_ops;
1626
1627 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
1628 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
1629 sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count);
1630 sit_i->sit_bitmap = dst_bitmap;
1631 sit_i->bitmap_size = bitmap_size;
1632 sit_i->dirty_sentries = 0;
1633 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
1634 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
1635 sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec;
1636 mutex_init(&sit_i->sentry_lock);
1637 return 0;
1638}
1639
1640static int build_free_segmap(struct f2fs_sb_info *sbi)
1641{
1642 struct f2fs_sm_info *sm_info = SM_I(sbi);
1643 struct free_segmap_info *free_i;
1644 unsigned int bitmap_size, sec_bitmap_size;
1645
1646 /* allocate memory for free segmap information */
1647 free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL);
1648 if (!free_i)
1649 return -ENOMEM;
1650
1651 SM_I(sbi)->free_info = free_i;
1652
1653 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));
1654 free_i->free_segmap = kmalloc(bitmap_size, GFP_KERNEL);
1655 if (!free_i->free_segmap)
1656 return -ENOMEM;
1657
1658 sec_bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi));
1659 free_i->free_secmap = kmalloc(sec_bitmap_size, GFP_KERNEL);
1660 if (!free_i->free_secmap)
1661 return -ENOMEM;
1662
1663 /* set all segments as dirty temporarily */
1664 memset(free_i->free_segmap, 0xff, bitmap_size);
1665 memset(free_i->free_secmap, 0xff, sec_bitmap_size);
1666
1667 /* init free segmap information */
1668 free_i->start_segno =
1669 (unsigned int) GET_SEGNO_FROM_SEG0(sbi, sm_info->main_blkaddr);
1670 free_i->free_segments = 0;
1671 free_i->free_sections = 0;
1672 rwlock_init(&free_i->segmap_lock);
1673 return 0;
1674}
1675
1676static int build_curseg(struct f2fs_sb_info *sbi)
1677{
1678 struct curseg_info *array;
1679 int i;
1680
Evan McClainf3f030d2014-07-17 21:16:35 -04001681 array = kcalloc(NR_CURSEG_TYPE, sizeof(*array), GFP_KERNEL);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001682 if (!array)
1683 return -ENOMEM;
1684
1685 SM_I(sbi)->curseg_array = array;
1686
1687 for (i = 0; i < NR_CURSEG_TYPE; i++) {
1688 mutex_init(&array[i].curseg_mutex);
1689 array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL);
1690 if (!array[i].sum_blk)
1691 return -ENOMEM;
1692 array[i].segno = NULL_SEGNO;
1693 array[i].next_blkoff = 0;
1694 }
1695 return restore_curseg_summaries(sbi);
1696}
1697
1698static void build_sit_entries(struct f2fs_sb_info *sbi)
1699{
1700 struct sit_info *sit_i = SIT_I(sbi);
1701 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1702 struct f2fs_summary_block *sum = curseg->sum_blk;
Changman Leeb1a94e82013-11-15 10:42:51 +09001703 int sit_blk_cnt = SIT_BLK_CNT(sbi);
1704 unsigned int i, start, end;
1705 unsigned int readed, start_blk = 0;
1706 int nrpages = MAX_BIO_BLOCKS(max_hw_blocks(sbi));
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001707
Changman Leeb1a94e82013-11-15 10:42:51 +09001708 do {
Chao Yu624b14f2014-02-07 16:11:53 +08001709 readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001710
Changman Leeb1a94e82013-11-15 10:42:51 +09001711 start = start_blk * sit_i->sents_per_block;
1712 end = (start_blk + readed) * sit_i->sents_per_block;
1713
1714 for (; start < end && start < TOTAL_SEGS(sbi); start++) {
1715 struct seg_entry *se = &sit_i->sentries[start];
1716 struct f2fs_sit_block *sit_blk;
1717 struct f2fs_sit_entry sit;
1718 struct page *page;
1719
1720 mutex_lock(&curseg->curseg_mutex);
1721 for (i = 0; i < sits_in_cursum(sum); i++) {
1722 if (le32_to_cpu(segno_in_journal(sum, i))
1723 == start) {
1724 sit = sit_in_journal(sum, i);
1725 mutex_unlock(&curseg->curseg_mutex);
1726 goto got_it;
1727 }
1728 }
1729 mutex_unlock(&curseg->curseg_mutex);
1730
1731 page = get_current_sit_page(sbi, start);
1732 sit_blk = (struct f2fs_sit_block *)page_address(page);
1733 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
1734 f2fs_put_page(page, 1);
1735got_it:
1736 check_block_count(sbi, start, &sit);
1737 seg_info_from_raw_sit(se, &sit);
1738 if (sbi->segs_per_sec > 1) {
1739 struct sec_entry *e = get_sec_entry(sbi, start);
1740 e->valid_blocks += se->valid_blocks;
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001741 }
1742 }
Changman Leeb1a94e82013-11-15 10:42:51 +09001743 start_blk += readed;
1744 } while (start_blk < sit_blk_cnt);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001745}
1746
1747static void init_free_segmap(struct f2fs_sb_info *sbi)
1748{
1749 unsigned int start;
1750 int type;
1751
1752 for (start = 0; start < TOTAL_SEGS(sbi); start++) {
1753 struct seg_entry *sentry = get_seg_entry(sbi, start);
1754 if (!sentry->valid_blocks)
1755 __set_free(sbi, start);
1756 }
1757
1758 /* set use the current segments */
1759 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
1760 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
1761 __set_test_and_inuse(sbi, curseg_t->segno);
1762 }
1763}
1764
1765static void init_dirty_segmap(struct f2fs_sb_info *sbi)
1766{
1767 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1768 struct free_segmap_info *free_i = FREE_I(sbi);
1769 unsigned int segno = 0, offset = 0, total_segs = TOTAL_SEGS(sbi);
1770 unsigned short valid_blocks;
1771
1772 while (1) {
1773 /* find dirty segment based on free segmap */
1774 segno = find_next_inuse(free_i, total_segs, offset);
1775 if (segno >= total_segs)
1776 break;
1777 offset = segno + 1;
1778 valid_blocks = get_valid_blocks(sbi, segno, 0);
1779 if (valid_blocks >= sbi->blocks_per_seg || !valid_blocks)
1780 continue;
1781 mutex_lock(&dirty_i->seglist_lock);
1782 __locate_dirty_segment(sbi, segno, DIRTY);
1783 mutex_unlock(&dirty_i->seglist_lock);
1784 }
1785}
1786
1787static int init_victim_secmap(struct f2fs_sb_info *sbi)
1788{
1789 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1790 unsigned int bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi));
1791
1792 dirty_i->victim_secmap = kzalloc(bitmap_size, GFP_KERNEL);
1793 if (!dirty_i->victim_secmap)
1794 return -ENOMEM;
1795 return 0;
1796}
1797
1798static int build_dirty_segmap(struct f2fs_sb_info *sbi)
1799{
1800 struct dirty_seglist_info *dirty_i;
1801 unsigned int bitmap_size, i;
1802
1803 /* allocate memory for dirty segments list information */
1804 dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL);
1805 if (!dirty_i)
1806 return -ENOMEM;
1807
1808 SM_I(sbi)->dirty_info = dirty_i;
1809 mutex_init(&dirty_i->seglist_lock);
1810
1811 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));
1812
1813 for (i = 0; i < NR_DIRTY_TYPE; i++) {
1814 dirty_i->dirty_segmap[i] = kzalloc(bitmap_size, GFP_KERNEL);
1815 if (!dirty_i->dirty_segmap[i])
1816 return -ENOMEM;
1817 }
1818
1819 init_dirty_segmap(sbi);
1820 return init_victim_secmap(sbi);
1821}
1822
1823/*
1824 * Update min, max modified time for cost-benefit GC algorithm
1825 */
1826static void init_min_max_mtime(struct f2fs_sb_info *sbi)
1827{
1828 struct sit_info *sit_i = SIT_I(sbi);
1829 unsigned int segno;
1830
1831 mutex_lock(&sit_i->sentry_lock);
1832
1833 sit_i->min_mtime = LLONG_MAX;
1834
1835 for (segno = 0; segno < TOTAL_SEGS(sbi); segno += sbi->segs_per_sec) {
1836 unsigned int i;
1837 unsigned long long mtime = 0;
1838
1839 for (i = 0; i < sbi->segs_per_sec; i++)
1840 mtime += get_seg_entry(sbi, segno + i)->mtime;
1841
1842 mtime = div_u64(mtime, sbi->segs_per_sec);
1843
1844 if (sit_i->min_mtime > mtime)
1845 sit_i->min_mtime = mtime;
1846 }
1847 sit_i->max_mtime = get_mtime(sbi);
1848 mutex_unlock(&sit_i->sentry_lock);
1849}
1850
1851int build_segment_manager(struct f2fs_sb_info *sbi)
1852{
1853 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
1854 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1855 struct f2fs_sm_info *sm_info;
1856 int err;
1857
1858 sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL);
1859 if (!sm_info)
1860 return -ENOMEM;
1861
1862 /* init sm info */
1863 sbi->sm_info = sm_info;
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001864 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
1865 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
1866 sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
1867 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
1868 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
1869 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
1870 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
Jaegeuk Kimf183b112014-03-19 14:17:21 +09001871 sm_info->rec_prefree_segments = sm_info->main_segments *
1872 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
Changman Leeb1a94e82013-11-15 10:42:51 +09001873 sm_info->ipu_policy = F2FS_IPU_DISABLE;
1874 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
1875
1876 INIT_LIST_HEAD(&sm_info->discard_list);
1877 sm_info->nr_discards = 0;
1878 sm_info->max_discards = 0;
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001879
Evan McClainf3f030d2014-07-17 21:16:35 -04001880 if (test_opt(sbi, FLUSH_MERGE) && !f2fs_readonly(sbi->sb)) {
1881 err = create_flush_cmd_control(sbi);
1882 if (err)
1883 return err;
1884 }
1885
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001886 err = build_sit_info(sbi);
1887 if (err)
1888 return err;
1889 err = build_free_segmap(sbi);
1890 if (err)
1891 return err;
1892 err = build_curseg(sbi);
1893 if (err)
1894 return err;
1895
1896 /* reinit free segmap based on SIT */
1897 build_sit_entries(sbi);
1898
1899 init_free_segmap(sbi);
1900 err = build_dirty_segmap(sbi);
1901 if (err)
1902 return err;
1903
1904 init_min_max_mtime(sbi);
1905 return 0;
1906}
1907
1908static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
1909 enum dirty_type dirty_type)
1910{
1911 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1912
1913 mutex_lock(&dirty_i->seglist_lock);
1914 kfree(dirty_i->dirty_segmap[dirty_type]);
1915 dirty_i->nr_dirty[dirty_type] = 0;
1916 mutex_unlock(&dirty_i->seglist_lock);
1917}
1918
1919static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
1920{
1921 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1922 kfree(dirty_i->victim_secmap);
1923}
1924
1925static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
1926{
1927 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1928 int i;
1929
1930 if (!dirty_i)
1931 return;
1932
1933 /* discard pre-free/dirty segments list */
1934 for (i = 0; i < NR_DIRTY_TYPE; i++)
1935 discard_dirty_segmap(sbi, i);
1936
1937 destroy_victim_secmap(sbi);
1938 SM_I(sbi)->dirty_info = NULL;
1939 kfree(dirty_i);
1940}
1941
1942static void destroy_curseg(struct f2fs_sb_info *sbi)
1943{
1944 struct curseg_info *array = SM_I(sbi)->curseg_array;
1945 int i;
1946
1947 if (!array)
1948 return;
1949 SM_I(sbi)->curseg_array = NULL;
1950 for (i = 0; i < NR_CURSEG_TYPE; i++)
1951 kfree(array[i].sum_blk);
1952 kfree(array);
1953}
1954
1955static void destroy_free_segmap(struct f2fs_sb_info *sbi)
1956{
1957 struct free_segmap_info *free_i = SM_I(sbi)->free_info;
1958 if (!free_i)
1959 return;
1960 SM_I(sbi)->free_info = NULL;
1961 kfree(free_i->free_segmap);
1962 kfree(free_i->free_secmap);
1963 kfree(free_i);
1964}
1965
1966static void destroy_sit_info(struct f2fs_sb_info *sbi)
1967{
1968 struct sit_info *sit_i = SIT_I(sbi);
1969 unsigned int start;
1970
1971 if (!sit_i)
1972 return;
1973
1974 if (sit_i->sentries) {
1975 for (start = 0; start < TOTAL_SEGS(sbi); start++) {
1976 kfree(sit_i->sentries[start].cur_valid_map);
1977 kfree(sit_i->sentries[start].ckpt_valid_map);
1978 }
1979 }
1980 vfree(sit_i->sentries);
1981 vfree(sit_i->sec_entries);
1982 kfree(sit_i->dirty_sentries_bitmap);
1983
1984 SM_I(sbi)->sit_info = NULL;
1985 kfree(sit_i->sit_bitmap);
1986 kfree(sit_i);
1987}
1988
1989void destroy_segment_manager(struct f2fs_sb_info *sbi)
1990{
1991 struct f2fs_sm_info *sm_info = SM_I(sbi);
Evan McClainf3f030d2014-07-17 21:16:35 -04001992
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001993 if (!sm_info)
1994 return;
Evan McClainf3f030d2014-07-17 21:16:35 -04001995 destroy_flush_cmd_control(sbi);
Linus Torvalds8005ecc2012-12-20 13:54:51 -08001996 destroy_dirty_segmap(sbi);
1997 destroy_curseg(sbi);
1998 destroy_free_segmap(sbi);
1999 destroy_sit_info(sbi);
2000 sbi->sm_info = NULL;
2001 kfree(sm_info);
2002}
Changman Leeb1a94e82013-11-15 10:42:51 +09002003
2004int __init create_segment_manager_caches(void)
2005{
2006 discard_entry_slab = f2fs_kmem_cache_create("discard_entry",
Gu Zhenge33dcea2014-03-07 18:43:28 +08002007 sizeof(struct discard_entry));
Changman Leeb1a94e82013-11-15 10:42:51 +09002008 if (!discard_entry_slab)
2009 return -ENOMEM;
2010 return 0;
2011}
2012
2013void destroy_segment_manager_caches(void)
2014{
2015 kmem_cache_destroy(discard_entry_slab);
2016}